keyword/code/Second_code.py

import jieba
from collections import Counter
import copy
import pandas as pd

#一.中介关键词
def file(seedword,filename):
    init_data = open('./result.train','r',encoding='utf-8')
    result_data = open(filename,'w',encoding='utf-8')
    for line in init_data:
        if seedword in line:
            result_data.write(line)
    init_data.close()
    result_data.close()

seedwords=['图片','手机','小说','视频','下载','qq','电影','百度','英语','游戏']
for seedword in seedwords:
    filename = './seedword_'+seedword
    file(seedword,filename)


def seedwords_segmentation(seedword):
    file = './seedword_'+seedword
    segmentation_file = './seedword_segmentation_'+seedword
    seedwords_data = open(file,'r',encoding='utf-8')
    segmentation_data = open(segmentation_file,'w',encoding='utf-8')
    for x in seedwords_data:
        x = x[:-1]
        word_seg = jieba.cut(x)
        line_string = '\t'.join(word_seg)+'\n'
        segmentation_data.write(line_string)
    seedwords_data.close()
    segmentation_data.close()


for seedword in seedwords:
    seedwords_segmentation(seedword)

stop_word=set(['我','都','与','你','多少','年','月','什么','及',
                    '和','上','如何','是','吗','有','在','可以','什么','用','了','说',
                    '大','很','版','之','为什么','怎么办','能','玩','级','写',',',' ','的','怎么'])
def stop_words_filter(words):
    word_cleaned = []
    for word in words:
        if word not in stop_word:
            word_cleaned.append(word)
    return word_cleaned

def getwordlist(seedword):
    wordlist = [];
    segmentation_file = './seedword_segmentation_'+seedword
    seedwords_data = open(segmentation_file,'r',encoding='utf-8')
    for line in seedwords_data:
        line = line[:-1]
        words = line.split('\t')
        wordlist.extend(words)
    seedwords_data.close()
    return wordlist


def word_frequency_count(seedword):
    word_list = getwordlist(seedword)
    word_list = stop_words_filter(word_list)
    count_result = Counter(word_list)
    midkeywords=[]
    flag = False
    for key, val in count_result.most_common(12):
        if flag==True:
            midkeywords.append(key)
        flag=True
    return midkeywords


#得到种子关键词的候选中介关键词集合
midkeywords_list = []
for seedword in seedwords:
    midkeywords_list.append(word_frequency_count(seedword))

#print("midkeywords_list")
print(midkeywords_list)

#midkeywords_list[0].remove('图片')
#midkeywords_list[1].remove('手机')
#midkeywords_list[2].remove('小说')
#midkeywords_list[3].remove('视频')
#midkeywords_list[4].remove('下载')
#midkeywords_list[5].remove('qq')
#midkeywords_list[6].remove('电影')
#midkeywords_list[7].remove('百度')
#midkeywords_list[8].remove('英语')
#midkeywords_list[9].remove('游戏')



def seedword_query_volume(seedword):
    word_list = getwordlist(seedword)
    count_result = Counter(word_list)
    for key, val in count_result.most_common(1):
         return val

s_query_volume = {}
for seedword in seedwords:
    s_query_volume[seedword] = seedword_query_volume(seedword)



# 得到10个种子关键词对应中介关键词，存在midkeywords这个字典中
midkeywords=dict(zip(seedwords,midkeywords_list))
midkeywords


#二. 搜索量统计

def query_volume_count(seedword):
    count_list = []
    filename = './seedword_'+seedword
    for midkeyword in midkeywords[seedword]:
        count = 0
        init_data = open(filename,'r',encoding = 'utf-8')
        for setence in init_data:
            if seedword in setence and midkeyword in setence:
                count+=1
        count_list.append(count)
        init_data.close()
    return count_list

# 统计得到的搜索量sa,存储在字典sa_query_volume中
sa_query_volume = {}
for seedword in seedwords:
    sa_count_list=[]
    sa_count_list = query_volume_count(seedword)
    sa_count_dict = dict(zip(midkeywords[seedword],sa_count_list))
    sa_query_volume[seedword]=sa_count_dict
sa_query_volume


#三. 确定中介关键字的权重
def count_midkeyword_weight():
    midkeyword_weight={}
    for seedword in seedwords:
        s = s_query_volume[seedword]
        weight_dict = {}
        for midkeyword in sa_query_volume[seedword]:
            sa = sa_query_volume[seedword][midkeyword]
            weight = sa/s
            weight_dict[midkeyword] = weight
        midkeyword_weight[seedword] = weight_dict
    return midkeyword_weight



# 将中介关键词的权重结果存储在字典w_midkeyword中
w_midkeyword = dict(count_midkeyword_weight())
w_midkeyword


#四.竞争性关键字集合

def comkey_words_file(seedword):
    init_data = open('./words2.train','r',encoding='utf-8')
    filename = './comkey_'+seedword
    result_data = open(filename,'w',encoding='utf-8')
    for line in init_data:
        if seedword not in line:
            for midkeyword in midkeywords[seedword]:
                if midkeyword in line:
                    result_data.write(line)
                    break
    init_data.close()
    result_data.close()
    
for seedword in seedwords:
    comkey_words_file(seedword)


def get_comkey_words(seedword):
    for midkeyword in midkeywords[seedword]:
        comkey_data = open('./comkey_'+seedword,'r',encoding='utf-8')
        comkeyquery_list = []
        for line in comkey_data:
            if midkeyword in line:
                line = line[:-1]
                word_seg = jieba.cut(line)
                comkeyquery_list.extend(word_seg)
        comkeyquery_list=stop_words_filter(comkeyquery_list)
        count_result = Counter(comkeyquery_list)
        for key, val in count_result.most_common(3):
            if key!=midkeyword:
                print(key,val,end='  ')
        comkey_data.close()



for seedword in seedwords:
    print(seedword+':\n')
    get_comkey_words(seedword)



# 竞争性关键词筛选
# 同时得到|{ka}|的值
ka_list=[[12907,2078,1495,8434,3095,4896,12021,10427],
         [34432,10393,7775,3805,5295,1592,2824,1021],
         [7135,2128,2852,2050,22595,2227,3792,33049],
         [832,17400,17304,31527,11673,1038,1640,29],
         [2059,10909,5397,6621,14470,4107,8198,35225],
         [11313,4737,2172,16375,2050,898,3270,1967],
         [876,66592,1011,18240,7890,18840,11613,31519],
         [11794,11417,2717,34264,205,317,1723,2724],
         [3351,10414,6072,2133,9202,401,3577,2921],
         [34412,11511,31527,967,3883,5253,41,6798]]

comp_list=[['做法','男生','句子','价格表','视频','最新','下载','头像'],
          ['txt','头像','苹果7','下载','荣耀','大全','路由器','高清'],
          ['穿越火线','电影','前十名','耽美','全文','末世','女主角','txt'],
          ['设计','观看','在线','图片','下载','游戏','大全','教学'],
          ['重生','qq','电视剧','手机','阅读','手机游戏','穿越','百度云'],
          ['下载','女生','重生','查询','侠盗','163邮箱','背景音乐','字'],
          ['中国','官网','视频','在线','下载','观看','qq','图片'],
          ['青云志','贴吧','人力资源','txt','糗事','视频','360云盘','中国地图'],
          ['语文','字','阅读','英文单词','大学生','成绩','英文翻译','视频教程'],
          ['txt','qq','图片','单机版','地方','大全','exo','笔记本电脑']]



compkeywords=dict(zip(seedwords,comp_list))
compkeywords

# 每个中介关键词对应的竞争性关键词
compwords=dict.fromkeys(seedwords,{})
i=0
for list in midkeywords_list: 
    compwords[seedwords[i]]=dict(zip(list,comp_list[i]))
    i+=1



# 将|{ka}|的值存在字典ka_query_list中
ka_query_volume=dict.fromkeys(seedwords,{})
i=0
for list in midkeywords_list: 
    ka_query_volume[seedwords[i]]=dict(zip(list,ka_list[i]))
    i+=1
ka_query_volume


#五.计算竞争性关键字的竞争度


def count_midkeyword(seedword):
    count_dict=dict.fromkeys(midkeywords[seedword],0)
    init_data = open('./words2.train','r',encoding = 'utf-8')
    for sentence in init_data: 
        for midkeyword in midkeywords[seedword]:
            if midkeyword in sentence:
                count_dict[midkeyword]+=1
    init_data.close()
    return count_dict
a_query_volume={}
for seedword in seedwords:
    a_query_volume[seedword]=count_midkeyword(seedword)



def getcomp():
    comp_query_volume = copy.deepcopy(ka_query_volume)
    for seedword in ka_query_volume:
        for midkeyword in ka_query_volume[seedword]:
             comp_query_volume[seedword][midkeyword] = ka_query_volume[seedword][midkeyword]/(a_query_volume[seedword][midkeyword]-sa_query_volume[seedword][midkeyword])
    return comp_query_volume  
result_query_volume=getcomp()
result_query_volume


def comp_result():
    result={}
    for seedword in result_query_volume:
        result.setdefault(seedword,{})
        i=0
        for midkeyword in result_query_volume[seedword]:
            result[seedword][compkeywords[seedword][i]]=w_midkeyword[seedword][midkeyword]*result_query_volume[seedword][midkeyword]
            i+=1
    return result
comp_k_s=comp_result()
#关键词k与种子关键词s的竞争性程度
comp_k_s


#六.竞争性关键字排序输出
def comp_print(seedword):
    resultpd= pd.DataFrame.from_dict(comp_k_s[seedword],orient='index',columns=['竞争度'])
    print(resultpd.sort_values(by='竞争度',ascending=False))

comp_print('图片')#图片的竞争性关键字
comp_print('手机')
comp_print('小说')
comp_print('视频')
comp_print('下载')
comp_print('qq')
comp_print('电影')
comp_print('百度')
comp_print('英语')
comp_print('游戏')