diff --git a/models/dbsql/dataset_foreigntable_for_es.sql b/models/dbsql/dataset_foreigntable_for_es.sql new file mode 100644 index 000000000..9e1e886b6 --- /dev/null +++ b/models/dbsql/dataset_foreigntable_for_es.sql @@ -0,0 +1,152 @@ +DROP FOREIGN TABLE public.dataset_es; +CREATE FOREIGN TABLE public.dataset_es +( + id bigint NOT NULL, + title character varying(255), + status integer, + category character varying(255), + description text, + download_times bigint, + license character varying(255), + task character varying(255), + release_id bigint, + user_id bigint, + repo_id bigint, + created_unix bigint, + updated_unix bigint, + file_name text +)SERVER multicorn_es +OPTIONS + ( + host '192.168.207.94', + port '9200', + index 'user_es-index', + rowid_column 'id', + default_sort '_id' + ) +; +DELETE FROM public.dataset_es; + INSERT INTO public.dataset_es( + id, + title, + status, + category, + description, + download_times, + license, task, + release_id, + user_id, + repo_id, + created_unix, + updated_unix,file_name) + SELECT + id, + title, + status, + category, + description, + download_times, + license, + task, + release_id, + user_id, + repo_id, + created_unix, + updated_unix,(select array_to_string(array_agg(name order by created_unix desc),',') from public.attachment a where a.dataset_id=b.id) + FROM public.dataset b; + +CREATE OR REPLACE FUNCTION public.insert_dataset_data() RETURNS trigger AS +$def$ + BEGIN + INSERT INTO public.dataset_es( + id, + title, + status, + category, + description, + download_times, + license, task, + release_id, + user_id, + repo_id, + created_unix, + updated_unix) + VALUES ( + NEW.id, + NEW.title, + NEW.status, + NEW.category, + NEW.description, + NEW.download_times, + NEW.license, + NEW.task, + NEW.release_id, + NEW.user_id, + NEW.repo_id, + NEW.created_unix, + NEW.updated_unix + ); + RETURN NEW; + END; +$def$ +LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS es_insert_dataset on public.dataset; + +CREATE TRIGGER es_insert_dataset + AFTER INSERT ON public.dataset + FOR EACH ROW EXECUTE PROCEDURE insert_dataset_data(); + + +CREATE OR REPLACE FUNCTION public.udpate_dataset_file_name_delete() RETURNS trigger AS +$def$ + BEGIN + update public.dataset_es SET file_name=(select array_to_string(array_agg(name order by created_unix desc),',') from public.attachment where dataset_id=OLD.dataset_id) where id=OLD.dataset_id; + return NEW; + END; +$def$ +LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS es_udpate_dataset_file_name_delete on public.attachment; +CREATE TRIGGER es_udpate_dataset_file_name_delete + AFTER DELETE ON public.attachment + FOR EACH ROW EXECUTE PROCEDURE udpate_dataset_file_name_delete(); + + +CREATE OR REPLACE FUNCTION public.update_dataset() RETURNS trigger AS +$def$ + BEGIN + UPDATE public.dataset_es + SET description=NEW.description, + title=NEW.title, + category=NEW.category, + file_name=(select array_to_string(array_agg(name order by created_unix desc),',') from public.attachment where dataset_id=NEW.id) + where id=NEW.id; + return new; + END +$def$ +LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS es_update_dataset on public.dataset; + +CREATE TRIGGER es_update_dataset + AFTER UPDATE ON public.dataset + FOR EACH ROW EXECUTE PROCEDURE update_dataset(); + + +CREATE OR REPLACE FUNCTION public.delete_dataset() RETURNS trigger AS +$def$ + declare + BEGIN + DELETE FROM public.dataset_es where id=OLD.id; + return new; + END +$def$ +LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS es_delete_dataset on public.dataset; +CREATE TRIGGER es_delete_dataset + AFTER DELETE ON public.dataset + FOR EACH ROW EXECUTE PROCEDURE delete_dataset(); + + diff --git a/models/dbsql/dateset_foreigntable_for_es.sql b/models/dbsql/dateset_foreigntable_for_es.sql deleted file mode 100644 index 55522af3b..000000000 --- a/models/dbsql/dateset_foreigntable_for_es.sql +++ /dev/null @@ -1,155 +0,0 @@ -CREATE FOREIGN TABLE public."dataset_es" -( - id bigint NOT NULL, - title character varying(255), - status integer, - category character varying(255), - description text, - download_times bigint, - license character varying(255), - task character varying(255), - release_id bigint, - user_id bigint, - repo_id bigint, - created_unix bigint, - updated_unix bigint, - file_name text -)SERVER multicorn_es -OPTIONS - ( - host '192.168.207.94', - port '9200', - index 'user_es-index', - rowid_column 'id' - ) -; - -CREATE OR REPLACE FUNCTION public.insert_dataset_data() RETURNS trigger AS -$def$ - BEGIN - INSERT INTO public.dataset_es( - id, - title, - status, - category, - description, - download_times, - license, task, - release_id, - user_id, - repo_id, - created_unix, - updated_unix) - VALUES ( - NEW.id, - NEW.title, - NEW.status, - NEW.category, - NEW.description, - NEW.download_times, - NEW.license, - NEW.task, - NEW.release_id, - NEW.user_id, - NEW.repo_id, - NEW.created_unix, - NEW.updated_unix - ); - RETURN NEW; - END; -$def$ -LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS es_insert_dataset on public.dataset; - -CREATE TRIGGER es_insert_dataset - AFTER INSERT ON public.dataset - FOR EACH ROW EXECUTE PROCEDURE insert_dataset_data(); - -CREATE OR REPLACE FUNCTION public.udpate_dataset_file_name() RETURNS trigger AS -$def$ - BEGIN - if (TG_OP = 'DELETE') then - update public.dataset_es SET file_name=(select array_to_string(array_agg(name order by created_unix desc),',') from public.attachment where dataset_id=OLD.dataset_id order by created_unix desc) where id=OLD.dataset_id; - elsif (TG_OP = 'INSERT') then - update public.dataset_es SET file_name=(select array_to_string(array_agg(name order by created_unix desc),',') from public.attachment where dataset_id=NEW.dataset_id order by created_unix desc) where id=NEW.dataset_id; - end if; - - return null; - END; -$def$ -LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS es_udpate_dataset_file_name on public.attachment; -CREATE TRIGGER es_udpate_dataset_file_name - AFTER INSERT OR DELETE ON public.attachment - FOR EACH ROW EXECUTE PROCEDURE udpate_dataset_file_name(); - - -CREATE OR REPLACE FUNCTION public.update_dataset_description() RETURNS trigger AS -$def$ - declare - BEGIN - UPDATE public.dataset_es SET description=NEW.description where id=NEW.id; - return new; - END -$def$ -LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS es_update_dataset_description on public.dataset; - -CREATE TRIGGER es_update_dataset_description - AFTER UPDATE OF "description" ON public.dataset - FOR EACH ROW EXECUTE PROCEDURE update_dataset_description(); - - -CREATE OR REPLACE FUNCTION public.update_dataset_title() RETURNS trigger AS -$def$ - declare - BEGIN - UPDATE public.dataset_es SET title=NEW.title where id=NEW.id; - return new; - END -$def$ -LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS es_update_dataset_title on public.dataset; - -CREATE TRIGGER es_update_dataset_title - AFTER UPDATE OF "title" ON public.dataset - FOR EACH ROW EXECUTE PROCEDURE update_dataset_title(); - - -CREATE OR REPLACE FUNCTION public.update_dataset_category() RETURNS trigger AS -$def$ - declare - BEGIN - UPDATE public.dataset_es SET category=NEW.category where id=NEW.id; - return new; - END -$def$ -LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS es_update_dataset_category on public.dataset; - -CREATE TRIGGER es_update_dataset_category - AFTER UPDATE OF "category" ON public.dataset - FOR EACH ROW EXECUTE PROCEDURE update_dataset_category(); - - -CREATE OR REPLACE FUNCTION public.delete_dataset() RETURNS trigger AS -$def$ - declare - BEGIN - DELETE FROM public.dataset_es where id=OLD.id; - return new; - END -$def$ -LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS es_delete_dataset on public.dataset; -CREATE TRIGGER es_delete_dataset - AFTER DELETE ON public.dataset - FOR EACH ROW EXECUTE PROCEDURE delete_dataset(); - - diff --git a/models/dbsql/issue_foreigntable_for_es.sql b/models/dbsql/issue_foreigntable_for_es.sql index db235c3b5..d05dae93a 100644 --- a/models/dbsql/issue_foreigntable_for_es.sql +++ b/models/dbsql/issue_foreigntable_for_es.sql @@ -1,3 +1,4 @@ +DROP FOREIGN TABLE public.issue_es; CREATE FOREIGN TABLE public.issue_es ( id bigint NOT NULL, @@ -29,10 +30,60 @@ OPTIONS host '192.168.207.94', port '9200', index 'user_es-index', - rowid_column 'id' + rowid_column 'id', + default_sort '_id' ) ; +DELETE FROM public.issue_es; +INSERT INTO public.issue_es( + id, + repo_id, + index, + poster_id, + original_author, + original_author_id, + name, + content, + milestone_id, + priority, + is_closed, + is_pull, + num_comments, + ref, + deadline_unix, + created_unix, + updated_unix, + closed_unix, + is_locked, + amount, + is_transformed,comment) + SELECT + id, + repo_id, + index, + poster_id, + original_author, + original_author_id, + name, + content, + milestone_id, + priority, + is_closed, + is_pull, + num_comments, + ref, + deadline_unix, + created_unix, + updated_unix, + closed_unix, + is_locked, + amount, + is_transformed, + (select array_to_string(array_agg(content order by created_unix desc),',') from public.comment a where a.issue_id=b.id) + FROM public.issue b; + + CREATE OR REPLACE FUNCTION public.insert_issue_data() RETURNS trigger AS $def$ BEGIN @@ -79,7 +130,7 @@ $def$ NEW.closed_unix, NEW.is_locked, NEW.amount, - NEW.is_transformed + NEW.is_transformed ); RETURN NEW; @@ -100,9 +151,7 @@ $def$ BEGIN if (TG_OP = 'DELETE') then update public.issue_es SET comment=(select array_to_string(array_agg(content order by created_unix desc),',') from public.comment where issue_id=OLD.issue_id) where id=OLD.issue_id; - elsif (TG_OP = 'INSERT') then - update public.issue_es SET comment=(select array_to_string(array_agg(content order by created_unix desc),',') from public.comment where issue_id=NEW.issue_id) where id=NEW.issue_id; - elsif (TG_OP = 'UPDATE') then + elsif (TG_OP = 'UPDATE') then update public.issue_es SET comment=(select array_to_string(array_agg(content order by created_unix desc),',') from public.comment where issue_id=NEW.issue_id) where id=NEW.issue_id; end if; @@ -113,75 +162,32 @@ LANGUAGE plpgsql; DROP TRIGGER IF EXISTS es_udpate_issue_comment on public.comment; CREATE TRIGGER es_udpate_issue_comment - AFTER INSERT OR DELETE OR UPDATE ON public.comment + AFTER DELETE OR UPDATE ON public.comment FOR EACH ROW EXECUTE PROCEDURE udpate_issue_comment(); -CREATE OR REPLACE FUNCTION public.update_issue_content() RETURNS trigger AS -$def$ - declare - BEGIN - UPDATE public.issue_es SET content=NEW.content where id=NEW.id; - return new; - END -$def$ -LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS es_update_issue_content on public.issue; - -CREATE TRIGGER es_update_issue_content - AFTER UPDATE OF "content" ON public.issue - FOR EACH ROW EXECUTE PROCEDURE update_issue_content(); - - -CREATE OR REPLACE FUNCTION public.update_issue_name() RETURNS trigger AS -$def$ - declare - BEGIN - UPDATE public.issue_es SET name=NEW.name where id=NEW.id; - return new; - END -$def$ -LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS es_update_issue_name on public.issue; - -CREATE TRIGGER es_update_issue_name - AFTER UPDATE OF "name" ON public.issue - FOR EACH ROW EXECUTE PROCEDURE update_issue_name(); - - -CREATE OR REPLACE FUNCTION public.update_issue_is_closed() RETURNS trigger AS +CREATE OR REPLACE FUNCTION public.update_issue() RETURNS trigger AS $def$ declare BEGIN - UPDATE public.issue_es SET is_closed=NEW.is_closed where id=NEW.id; + UPDATE public.issue_es + SET content=NEW.content, + name=NEW.name, + is_closed=NEW.is_closed, + num_comments=NEW.num_comments, + comment=(select array_to_string(array_agg(content order by created_unix desc),',') from public.comment where issue_id=NEW.id) + where id=NEW.id; return new; END $def$ LANGUAGE plpgsql; -DROP TRIGGER IF EXISTS es_update_issue_is_closed on public.issue; - -CREATE TRIGGER es_update_issue_is_closed - AFTER UPDATE OF "is_closed" ON public.issue - FOR EACH ROW EXECUTE PROCEDURE update_issue_is_closed(); - -CREATE OR REPLACE FUNCTION public.update_issue_num_comments() RETURNS trigger AS -$def$ - declare - BEGIN - UPDATE public.issue_es SET num_comments=NEW.num_comments where id=NEW.id; - return new; - END -$def$ -LANGUAGE plpgsql; +DROP TRIGGER IF EXISTS es_update_issue on public.issue; -DROP TRIGGER IF EXISTS es_update_issue_num_comments on public.issue; +CREATE TRIGGER es_update_issue + AFTER UPDATE ON public.issue + FOR EACH ROW EXECUTE PROCEDURE update_issue(); -CREATE TRIGGER es_update_issue_num_comments - AFTER UPDATE OF "num_comments" ON public.issue - FOR EACH ROW EXECUTE PROCEDURE update_issue_num_comments(); CREATE OR REPLACE FUNCTION public.delete_issue() RETURNS trigger AS diff --git a/models/dbsql/pr_foreigntable_for_es.sql b/models/dbsql/pr_foreigntable_for_es.sql index 183844d87..1ebf4093b 100644 --- a/models/dbsql/pr_foreigntable_for_es.sql +++ b/models/dbsql/pr_foreigntable_for_es.sql @@ -1,3 +1,4 @@ +DROP FOREIGN TABLE public.pull_request_es; CREATE FOREIGN TABLE public.pull_request_es ( id bigint NOT NULL, @@ -25,10 +26,56 @@ OPTIONS host '192.168.207.94', port '9200', index 'user_es-index', - rowid_column 'id' + rowid_column 'id', + default_sort '_id' ) ; +delete from public.pull_request_es; + INSERT INTO public.pull_request_es( + id, + type, + status, + conflicted_files, + commits_ahead, + commits_behind, + issue_id, + index, + head_repo_id, + base_repo_id, + head_branch, + base_branch, + merge_base, + has_merged, + merged_commit_id, + merger_id, + merged_unix, + is_transformed, + amount) + SELECT + id, + type, + status, + conflicted_files, + commits_ahead, + commits_behind, + issue_id, + index, + head_repo_id, + base_repo_id, + head_branch, + base_branch, + merge_base, + has_merged, + merged_commit_id, + merger_id, + merged_unix, + is_transformed, + amount + FROM public.pull_request; + + + CREATE OR REPLACE FUNCTION public.insert_pull_request_data() RETURNS trigger AS $def$ BEGIN @@ -74,9 +121,6 @@ $def$ NEW.amount ); - --update issue - UPDATE public.issue_es SET pr_id=NEW.id where id=NEW.issue_id; - RETURN NEW; END; $def$ @@ -90,21 +134,23 @@ CREATE TRIGGER es_insert_pull_request -CREATE OR REPLACE FUNCTION public.update_pull_request_has_merged() RETURNS trigger AS +CREATE OR REPLACE FUNCTION public.update_pull_request() RETURNS trigger AS $def$ declare BEGIN - UPDATE public.pull_request_es SET has_merged=NEW.has_merged where id=NEW.id; + UPDATE public.pull_request_es + SET has_merged=NEW.has_merged + where id=NEW.id; return new; END $def$ LANGUAGE plpgsql; -DROP TRIGGER IF EXISTS es_update_pull_request_has_merged on public.pull_request; +DROP TRIGGER IF EXISTS es_update_pull_request on public.pull_request; -CREATE TRIGGER es_update_pull_request_has_merged - AFTER UPDATE OF "has_merged" ON public.pull_request - FOR EACH ROW EXECUTE PROCEDURE update_pull_request_has_merged(); +CREATE TRIGGER es_update_pull_request + AFTER UPDATE ON public.pull_request + FOR EACH ROW EXECUTE PROCEDURE update_pull_request(); CREATE OR REPLACE FUNCTION public.delete_pull_request() RETURNS trigger AS diff --git a/models/dbsql/repo_foreigntable_for_es.sql b/models/dbsql/repo_foreigntable_for_es.sql index 3637853cd..027a7de76 100644 --- a/models/dbsql/repo_foreigntable_for_es.sql +++ b/models/dbsql/repo_foreigntable_for_es.sql @@ -1,4 +1,4 @@ -DROP FOREIGN TABLE public.repository_es; +drop FOREIGN table if exists public.repository_es; CREATE FOREIGN TABLE public.repository_es ( id bigint NOT NULL, owner_id bigint, @@ -50,9 +50,96 @@ OPTIONS host '192.168.207.94', port '9200', index 'repository-es-index', - rowid_column 'id' + rowid_column 'id', + default_sort '_id' ) ; +delete from public.repository_es; + INSERT INTO public.repository_es (id, + owner_id, + owner_name, + lower_name, + name, + description, + website, + original_service_type, + original_url, + default_branch, + num_watches, + num_stars, + num_forks, + num_issues, + num_closed_issues, + num_pulls, + num_closed_pulls, + num_milestones, + num_closed_milestones, + is_private, + is_empty, + is_archived, + is_mirror, + status, + is_fork, + fork_id, + is_template, + template_id, + size, + is_fsck_enabled, + close_issues_via_commit_in_any_branch, + topics, + avatar, + created_unix, + updated_unix, + contract_address, + block_chain_status, + balance, + clone_cnt, + num_commit, + git_clone_cnt) + SELECT + id, + owner_id, + owner_name, + lower_name, + name, + description, + website, + original_service_type, + original_url, + default_branch, + num_watches, + num_stars, + num_forks, + num_issues, + num_closed_issues, + num_pulls, + num_closed_pulls, + num_milestones, + num_closed_milestones, + is_private, + is_empty, + is_archived, + is_mirror, + status, + is_fork, + fork_id, + is_template, + template_id, + size, + is_fsck_enabled, + close_issues_via_commit_in_any_branch, + topics, + avatar, + created_unix, + updated_unix, + contract_address, + block_chain_status, + balance, + clone_cnt, + num_commit, + git_clone_cnt + FROM public.repository; + CREATE OR REPLACE FUNCTION public.insert_repository_data() RETURNS trigger AS $def$ @@ -152,162 +239,30 @@ CREATE TRIGGER es_insert_repository FOR EACH ROW EXECUTE PROCEDURE insert_repository_data(); -CREATE OR REPLACE FUNCTION public.update_repository_description() RETURNS trigger AS -$def$ - declare - BEGIN - UPDATE public.repository_es SET description=NEW.description where id=NEW.id; - return new; - END -$def$ -LANGUAGE plpgsql; - - -DROP TRIGGER IF EXISTS es_update_repository_description on public.repository; -DROP TRIGGER IF EXISTS es_update_repository_name on public.repository; -DROP TRIGGER IF EXISTS es_update_repository_ownername on public.repository; -DROP TRIGGER IF EXISTS es_update_repository_website on public.repository; -DROP TRIGGER IF EXISTS es_update_repository_topics on public.repository; -DROP TRIGGER IF EXISTS es_update_repository_updated_unix on public.repository; -DROP TRIGGER IF EXISTS es_update_repository_num_watches on public.repository; -DROP TRIGGER IF EXISTS es_update_repository_num_stars on public.repository; -DROP TRIGGER IF EXISTS es_update_repository_num_forks on public.repository; -DROP TRIGGER IF EXISTS es_delete_repository on public.repository; - -CREATE TRIGGER es_update_repository_description - AFTER UPDATE OF "description" ON public.repository - FOR EACH ROW EXECUTE PROCEDURE update_repository_description(); - - -CREATE OR REPLACE FUNCTION public.update_repository_name() RETURNS trigger AS -$def$ - declare - BEGIN - UPDATE public.repository_es SET name=NEW.name,lower_name=NEW.lower_name where id=NEW.id; - return new; - END -$def$ -LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS es_update_repository_name on public.repository; -CREATE TRIGGER es_update_repository_name - AFTER UPDATE OF "name","lower_name" ON public.repository - FOR EACH ROW EXECUTE PROCEDURE update_repository_name(); - - -CREATE OR REPLACE FUNCTION public.update_repository_ownername() RETURNS trigger AS +CREATE OR REPLACE FUNCTION public.update_repository() RETURNS trigger AS $def$ - declare - BEGIN - UPDATE public.repository_es SET owner_name=NEW.owner_name where id=NEW.id; - return new; - END -$def$ -LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS es_update_repository_ownername on public.repository; -CREATE TRIGGER es_update_repository_ownername - AFTER UPDATE OF "owner_name" ON public.repository - FOR EACH ROW EXECUTE PROCEDURE update_repository_ownername(); - -CREATE OR REPLACE FUNCTION public.update_repository_website() RETURNS trigger AS -$def$ - declare - BEGIN - UPDATE public.repository_es SET website=NEW.website where id=NEW.id; - return new; - END -$def$ -LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS es_update_repository_website on public.repository; -CREATE TRIGGER es_update_repository_website - AFTER UPDATE OF "website" ON public.repository - FOR EACH ROW EXECUTE PROCEDURE update_repository_website(); - - -CREATE OR REPLACE FUNCTION public.update_repository_topics() RETURNS trigger AS -$def$ - declare - BEGIN - UPDATE public.repository_es SET topics=NEW.topics where id=NEW.id; - return new; - END -$def$ -LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS es_update_repository_topics on public.repository; -CREATE TRIGGER es_update_repository_topics - AFTER UPDATE OF "topics" ON public.repository - FOR EACH ROW EXECUTE PROCEDURE update_repository_topics(); - - - -CREATE OR REPLACE FUNCTION public.update_repository_updated_unix() RETURNS trigger AS -$def$ - declare - BEGIN - UPDATE public.repository_es SET updated_unix=NEW.updated_unix where id=NEW.id; - return new; - END -$def$ -LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS es_update_repository_updated_unix on public.repository; -CREATE TRIGGER es_update_repository_updated_unix - AFTER UPDATE OF "updated_unix" ON public.repository - FOR EACH ROW EXECUTE PROCEDURE update_repository_updated_unix(); - - -CREATE OR REPLACE FUNCTION public.update_repository_num_watches() RETURNS trigger AS -$def$ - declare - BEGIN - UPDATE public.repository_es SET num_watches=NEW.num_watches where id=NEW.id; - return new; - END -$def$ -LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS es_update_repository_num_watches on public.repository; -CREATE TRIGGER es_update_repository_num_watches - AFTER UPDATE OF "num_watches" ON public.repository - FOR EACH ROW EXECUTE PROCEDURE update_repository_num_watches(); - - -CREATE OR REPLACE FUNCTION public.update_repository_num_stars() RETURNS trigger AS -$def$ - declare - BEGIN - UPDATE public.repository_es SET num_stars=NEW.num_stars where id=NEW.id; - return new; - END -$def$ -LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS es_update_repository_num_stars on public.repository; -CREATE TRIGGER es_update_repository_num_stars - AFTER UPDATE OF "num_stars" ON public.repository - FOR EACH ROW EXECUTE PROCEDURE update_repository_num_stars(); - - - -CREATE OR REPLACE FUNCTION public.update_repository_num_forks() RETURNS trigger AS -$def$ - declare BEGIN - UPDATE public.repository_es SET num_forks=NEW.num_forks where id=NEW.id; + update public.repository_es SET description=NEW.description, + name=NEW.name, + lower_name=NEW.lower_name, + owner_name=NEW.owner_name, + website=NEW.website, + updated_unix=NEW.updated_unix, + num_watches=NEW.num_watches, + num_stars=NEW.num_stars, + num_forks=NEW.num_forks, + topics=NEW.topics + where id=NEW.id; return new; END $def$ LANGUAGE plpgsql; +DROP TRIGGER IF EXISTS es_update_repository on public.repository; +CREATE TRIGGER es_update_repository + AFTER UPDATE ON public.repository + FOR EACH ROW EXECUTE PROCEDURE update_repository(); -DROP TRIGGER IF EXISTS es_update_repository_num_forks on public.repository; -CREATE TRIGGER es_update_repository_num_forks - AFTER UPDATE OF "num_forks" ON public.repository - FOR EACH ROW EXECUTE PROCEDURE update_repository_num_forks(); - CREATE OR REPLACE FUNCTION public.delete_repository() RETURNS trigger AS $def$ diff --git a/models/dbsql/user_foreigntable_for_es.sql b/models/dbsql/user_foreigntable_for_es.sql index dc362fce3..5708a5b4c 100644 --- a/models/dbsql/user_foreigntable_for_es.sql +++ b/models/dbsql/user_foreigntable_for_es.sql @@ -1,4 +1,5 @@ -CREATE FOREIGN TABLE public."user_es" +DROP FOREIGN table if exists public.user_es; +CREATE FOREIGN TABLE public.user_es ( id bigint NOT NULL , lower_name character varying(255) NULL, @@ -55,9 +56,98 @@ OPTIONS host '192.168.207.94', port '9200', index 'user_es-index', - rowid_column 'id' + rowid_column 'id', + default_sort '_id' ) ; +delete from public.user_es; + INSERT INTO public.user_es( + id, + lower_name, + name, + full_name, + email, + keep_email_private, + email_notifications_preference, + must_change_password, + login_type, + login_source, + login_name, + type, + location, + website, + rands, + language, + description, + created_unix, + updated_unix, + last_login_unix, + last_repo_visibility, + max_repo_creation, + is_active, + is_restricted, + allow_git_hook, + allow_import_local, + allow_create_organization, + prohibit_login, + avatar, + avatar_email, + use_custom_avatar, + num_followers, + num_following, + num_stars, + num_repos, + num_teams, + num_members, + visibility, + repo_admin_change_team_access, + diff_view_style, + theme, + is_operator) + SELECT + id, + lower_name, + name, + full_name, + email, + keep_email_private, + email_notifications_preference, + must_change_password, + login_type, + login_source, + login_name, + type, + location, + website, + rands, + language, + description, + created_unix, + updated_unix, + last_login_unix, + last_repo_visibility, + max_repo_creation, + is_active, + is_restricted, + allow_git_hook, + allow_import_local, + allow_create_organization, + prohibit_login, + avatar, + avatar_email, + use_custom_avatar, + num_followers, + num_following, + num_stars, + num_repos, + num_teams, + num_members, + visibility, + repo_admin_change_team_access, + diff_view_style, + theme, + is_operator + FROM public.user; CREATE OR REPLACE FUNCTION public.insert_user_data() RETURNS trigger AS $def$ @@ -162,107 +252,27 @@ CREATE TRIGGER es_insert_user FOR EACH ROW EXECUTE PROCEDURE insert_user_data(); -CREATE OR REPLACE FUNCTION public.update_user_description() RETURNS trigger AS +CREATE OR REPLACE FUNCTION public.update_user() RETURNS trigger AS $def$ - declare - BEGIN - UPDATE public.user_es SET description=NEW.description where id=NEW.id; - return new; - END -$def$ -LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS es_update_user_description on public.user; - -CREATE TRIGGER es_update_user_description - AFTER UPDATE OF "description" ON public.user - FOR EACH ROW EXECUTE PROCEDURE update_user_description(); - - -CREATE OR REPLACE FUNCTION public.update_user_name() RETURNS trigger AS -$def$ - declare - BEGIN - UPDATE public.user_es SET name=NEW.name where id=NEW.id; - return new; - END -$def$ -LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS es_update_user_name on public.user; - -CREATE TRIGGER es_update_user_name - AFTER UPDATE OF "name" ON public.user - FOR EACH ROW EXECUTE PROCEDURE update_user_name(); - - -CREATE OR REPLACE FUNCTION public.update_user_full_name() RETURNS trigger AS -$def$ - declare - BEGIN - UPDATE public.user_es SET full_name=NEW.full_name where id=NEW.id; - return new; - END -$def$ -LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS es_update_user_full_name on public.user; - -CREATE TRIGGER es_update_user_full_name - AFTER UPDATE OF "full_name" ON public.user - FOR EACH ROW EXECUTE PROCEDURE update_user_full_name(); - - - -CREATE OR REPLACE FUNCTION public.update_user_location() RETURNS trigger AS -$def$ - declare - BEGIN - UPDATE public.user_es SET location=NEW.location where id=NEW.id; - return new; - END -$def$ -LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS es_update_user_location on public.user; - -CREATE TRIGGER es_update_user_location - AFTER UPDATE OF "location" ON public.user - FOR EACH ROW EXECUTE PROCEDURE update_user_location(); - - -CREATE OR REPLACE FUNCTION public.update_user_website() RETURNS trigger AS -$def$ - declare - BEGIN - UPDATE public.user_es SET website=NEW.website where id=NEW.id; - return new; - END -$def$ -LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS es_update_user_website on public.user; - -CREATE TRIGGER es_update_user_website - AFTER UPDATE OF "website" ON public.user - FOR EACH ROW EXECUTE PROCEDURE update_user_website(); - - -CREATE OR REPLACE FUNCTION public.update_user_email() RETURNS trigger AS -$def$ - declare BEGIN - UPDATE public.user_es SET email=NEW.email where id=NEW.id; + UPDATE public.user_es + SET description=NEW.description, + name=NEW.name, + full_name=NEW.full_name, + location=NEW.location, + website=NEW.website, + email=NEW.email + where id=NEW.id; return new; END $def$ LANGUAGE plpgsql; -DROP TRIGGER IF EXISTS es_update_user_email on public.user; +DROP TRIGGER IF EXISTS es_update_user on public.user; -CREATE TRIGGER es_update_user_email - AFTER UPDATE OF "email" ON public.user - FOR EACH ROW EXECUTE PROCEDURE update_user_email(); +CREATE TRIGGER es_update_user + AFTER UPDATE ON public.user + FOR EACH ROW EXECUTE PROCEDURE update_user(); diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index c958e47e6..a15b9db5f 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -201,15 +201,7 @@ func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusi return nil, 0 } log.Info("query return total:" + fmt.Sprint(allCount)) - if allCount == 0 { - CommitCodeSizeMap, err := GetAllUserKPIStats() - if err != nil { - log.Info("query commit code errr.") - } else { - log.Info("query commit code size, len=" + fmt.Sprint(len(CommitCodeSizeMap))) - } - RefreshUserStaticAllTabel(make(map[string]int), CommitCodeSizeMap) - } + pageSize := 1000 totalPage := int(allCount) / pageSize userBusinessAnalysisReturnList := UserBusinessAnalysisAllList{} @@ -369,7 +361,7 @@ func RefreshUserStaticAllTabel(wikiCountMap map[string]int, CommitCodeSizeMap ma CodeMergeCountMap := queryPullRequest(start_unix, end_unix) CommitCountMap := queryCommitAction(start_unix, end_unix, 5) - IssueCountMap := queryAction(start_unix, end_unix, 6) + IssueCountMap := queryCreateIssue(start_unix, end_unix) CommentCountMap := queryComment(start_unix, end_unix) FocusRepoCountMap := queryWatch(start_unix, end_unix) @@ -394,7 +386,7 @@ func RefreshUserStaticAllTabel(wikiCountMap map[string]int, CommitCodeSizeMap ma var indexTotal int64 indexTotal = 0 for { - sess.Select("`user`.*").Table("user").Where(cond).Limit(Page_SIZE, int(indexTotal)) + sess.Select("`user`.*").Table("user").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) userList := make([]*User, 0) sess.Find(&userList) for i, userRecord := range userList { @@ -527,7 +519,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, DataDate := startTime.Format("2006-01-02") CodeMergeCountMap := queryPullRequest(start_unix, end_unix) CommitCountMap := queryCommitAction(start_unix, end_unix, 5) - IssueCountMap := queryAction(start_unix, end_unix, 6) + IssueCountMap := queryCreateIssue(start_unix, end_unix) CommentCountMap := queryComment(start_unix, end_unix) FocusRepoCountMap := queryWatch(start_unix, end_unix) @@ -558,7 +550,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, var indexTotal int64 indexTotal = 0 for { - sess.Select("`user`.*").Table("user").Where(cond).Limit(Page_SIZE, int(indexTotal)) + sess.Select("`user`.*").Table("user").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) userList := make([]*User, 0) sess.Find(&userList) @@ -708,7 +700,7 @@ func querySolveIssue(start_unix int64, end_unix int64) map[int64]int { issueAssigneesList := make([]*IssueAssignees, 0) sess.Select("issue_assignees.*").Table("issue_assignees"). Join("inner", "issue", "issue.id=issue_assignees.issue_id"). - Where(cond).Limit(Page_SIZE, int(indexTotal)) + Where(cond).OrderBy("issue_assignees.id asc").Limit(Page_SIZE, int(indexTotal)) sess.Find(&issueAssigneesList) @@ -743,7 +735,7 @@ func queryPullRequest(start_unix int64, end_unix int64) map[int64]int { indexTotal = 0 for { issueList := make([]*Issue, 0) - sess.Select("issue.*").Table("issue").Join("inner", "pull_request", "issue.id=pull_request.issue_id").Where(cond).Limit(Page_SIZE, int(indexTotal)) + sess.Select("issue.*").Table("issue").Join("inner", "pull_request", "issue.id=pull_request.issue_id").Where(cond).OrderBy("issue.id asc").Limit(Page_SIZE, int(indexTotal)) sess.Find(&issueList) log.Info("query issue(PR) size=" + fmt.Sprint(len(issueList))) for _, issueRecord := range issueList { @@ -776,7 +768,7 @@ func queryCommitAction(start_unix int64, end_unix int64, actionType int64) map[i var indexTotal int64 indexTotal = 0 for { - sess.Select("id,user_id,op_type,act_user_id").Table("action").Where(cond).Limit(Page_SIZE, int(indexTotal)) + sess.Select("id,user_id,op_type,act_user_id").Table("action").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) actionList := make([]*Action, 0) sess.Find(&actionList) @@ -798,29 +790,30 @@ func queryCommitAction(start_unix int64, end_unix int64, actionType int64) map[i return resultMap } -func queryAction(start_unix int64, end_unix int64, actionType int64) map[int64]int { +func queryCreateIssue(start_unix int64, end_unix int64) map[int64]int { + sess := x.NewSession() defer sess.Close() resultMap := make(map[int64]int) - cond := "op_type=" + fmt.Sprint(actionType) + " and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + cond := "is_pull=false and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) - count, err := sess.Where(cond).Count(new(Action)) + count, err := sess.Where(cond).Count(new(Issue)) if err != nil { - log.Info("query Action error. return.") + log.Info("query Issue error. return.") return resultMap } var indexTotal int64 indexTotal = 0 for { - sess.Select("id,user_id,op_type,act_user_id").Table("action").Where(cond).Limit(Page_SIZE, int(indexTotal)) - actionList := make([]*Action, 0) - sess.Find(&actionList) - log.Info("query action size=" + fmt.Sprint(len(actionList))) - for _, actionRecord := range actionList { - if _, ok := resultMap[actionRecord.UserID]; !ok { - resultMap[actionRecord.UserID] = 1 + sess.Select("id,poster_id").Table("issue").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) + issueList := make([]*Issue, 0) + sess.Find(&issueList) + log.Info("query issue size=" + fmt.Sprint(len(issueList))) + for _, issueRecord := range issueList { + if _, ok := resultMap[issueRecord.PosterID]; !ok { + resultMap[issueRecord.PosterID] = 1 } else { - resultMap[actionRecord.UserID] += 1 + resultMap[issueRecord.PosterID] += 1 } } indexTotal += Page_SIZE @@ -829,6 +822,7 @@ func queryAction(start_unix int64, end_unix int64, actionType int64) map[int64]i } } return resultMap + } func queryComment(start_unix int64, end_unix int64) map[int64]int { @@ -845,7 +839,7 @@ func queryComment(start_unix int64, end_unix int64) map[int64]int { var indexTotal int64 indexTotal = 0 for { - sess.Select("id,type,poster_id").Table("comment").Where(cond).Limit(Page_SIZE, int(indexTotal)) + sess.Select("id,type,poster_id").Table("comment").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) commentList := make([]*Comment, 0) sess.Find(&commentList) log.Info("query Comment size=" + fmt.Sprint(len(commentList))) @@ -881,7 +875,7 @@ func queryWatch(start_unix int64, end_unix int64) map[int64]int { indexTotal = 0 for { watchList := make([]*Watch, 0) - sess.Select("id,user_id,repo_id").Table("watch").Where(cond).Limit(Page_SIZE, int(indexTotal)) + sess.Select("id,user_id,repo_id").Table("watch").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) sess.Find(&watchList) log.Info("query Watch size=" + fmt.Sprint(len(watchList))) @@ -919,7 +913,7 @@ func queryStar(start_unix int64, end_unix int64) map[int64]int { var indexTotal int64 indexTotal = 0 for { - sess.Select("id,uid,repo_id").Table("star").Where(cond).Limit(Page_SIZE, int(indexTotal)) + sess.Select("id,uid,repo_id").Table("star").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) starList := make([]*Star, 0) sess.Find(&starList) @@ -955,7 +949,7 @@ func queryFollow(start_unix int64, end_unix int64) map[int64]int { var indexTotal int64 indexTotal = 0 for { - sess.Select("id,user_id,follow_id").Table("follow").Where(cond).Limit(Page_SIZE, int(indexTotal)) + sess.Select("id,user_id,follow_id").Table("follow").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) followList := make([]*Follow, 0) sess.Find(&followList) @@ -991,7 +985,7 @@ func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int { var indexTotal int64 indexTotal = 0 for { - sess.Select("id,uploader_id,size").Table("attachment").Where(cond).Limit(Page_SIZE, int(indexTotal)) + sess.Select("id,uploader_id,size").Table("attachment").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) attachmentList := make([]*Attachment, 0) sess.Find(&attachmentList) @@ -1027,7 +1021,7 @@ func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int { var indexTotal int64 indexTotal = 0 for { - sess.Select("id,owner_id,name").Table("repository").Where(cond).Limit(Page_SIZE, int(indexTotal)) + sess.Select("id,owner_id,name").Table("repository").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) repoList := make([]*Repository, 0) sess.Find(&repoList) log.Info("query Repository size=" + fmt.Sprint(len(repoList))) @@ -1117,7 +1111,7 @@ func queryLoginCount(start_unix int64, end_unix int64) map[int64]int { var indexTotal int64 indexTotal = 0 for { - statictisSess.Select("id,u_id").Table("user_login_log").Where(cond).Limit(Page_SIZE, int(indexTotal)) + statictisSess.Select("id,u_id").Table("user_login_log").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) userLoginLogList := make([]*UserLoginLog, 0) statictisSess.Find(&userLoginLogList) log.Info("query user login size=" + fmt.Sprint(len(userLoginLogList))) diff --git a/routers/home.go b/routers/home.go index 7cc353ed8..3262aa823 100755 --- a/routers/home.go +++ b/routers/home.go @@ -518,19 +518,19 @@ func RecommendOrgFromPromote(ctx *context.Context) { recommendFromPromote(ctx, url) } -func recommendFromPromote(ctx *context.Context, url string) { +func recommendFromPromote(ctx *context.Context, url string) []string { resp, err := http.Get(url) if err != nil { log.Info("Get organizations url error=" + err.Error()) - ctx.ServerError("QueryTrainJobList:", err) - return + ctx.ServerError("Get organizations url error:", err) + return nil } bytes, err := ioutil.ReadAll(resp.Body) resp.Body.Close() if err != nil { log.Info("Get organizations url error=" + err.Error()) - ctx.ServerError("QueryTrainJobList:", err) - return + ctx.ServerError("Read organizations url error:", err) + return nil } allLineStr := string(bytes) @@ -546,10 +546,11 @@ func recommendFromPromote(ctx *context.Context, url string) { result[i] = strings.Trim(line[tmpIndex+1:], " ") } } - ctx.JSON(http.StatusOK, result) + return result } func RecommendRepoFromPromote(ctx *context.Context) { url := setting.RecommentRepoAddr + "projects" recommendFromPromote(ctx, url) + }