import datetime import csv import pandas as pd import numpy as np import glob, os, re, time import matplotlib.pyplot as plt from matplotlib.ticker import FuncFormatter from difflib import SequenceMatcher from collections import Counter import difflib from docxtpl import DocxTemplate from docxtpl import InlineImage from docx.shared import Mm import jieba import jieba.posseg as pseg #--- #那我们的目标就是将字段列名的日期数据替换成标准的日期格式，具体的思路是： #1、先用excel实验2018-11-02对应的日期时间戳是43406。 #2、我再用2018-11-02减43406看看是从那一年开始计算的，所以得出结论是1899-12-30。 #3、那最后要达成目标就只需要时间戳+1899-12-30就等于对应的当前日 def ts2date(dates, sf='%Y-%m-%d'):#定义转化日期戳的函数,dates为日期戳 delta=datetime.timedelta(days=dates) today=datetime.datetime.strptime('1899-12-30','%Y-%m-%d')+delta#将1899-12-30转化为可以计算的时间格式并加上要转化的日期戳 return datetime.datetime.strftime(today,sf)#制定输出日期的格式 #--- def fetch_chinese(s): pattern =re.compile(r'[^\u4e00-\u9fa5]') sc = re.sub(pattern, '', s) return sc # 画柱状图 def drawBar(data, recipe, title='', fn=''): plt.figure(figsize=(6, 4)) plt.rcParams['font.sans-serif'] = ['SimHei'] plt.rcParams['axes.unicode_minus'] = False counties = recipe countyRates = data plt.bar(counties, countyRates, width=0.5) plt.xticks(counties, counties, rotation=35) plt.ylim((0, 1)) def to_percent(temp, position): return '%2.0f' % (100 * temp) + '%' plt.gca().yaxis.set_major_formatter(FuncFormatter(to_percent)) plt.title(title, fontsize=16) plt.tight_layout() plt.savefig(fn) # plt.show() plt.cla() plt.clf() plt.close() def getWBData(path, cities, hasBody=False): # cityShorten cityShorten = {'LZ':'兰州市', 'LX':'临夏回族自治州', 'JC':'金昌市', 'ZY':'张掖市', 'LN':'陇南市', 'JYG':'嘉峪关市', 'TS':'天水市', 'GN':'甘南藏族自治州', 'BY':'白银市', 'JQ':'酒泉市', 'QY':'庆阳市', 'PL':'平凉市', 'DX':'定西市', 'WW':'武威市', 'SZ':'省直部门', 'XQ': '兰州新区', 'LZXQ': '兰州新区', 'lz': '兰州市', 'lx': '临夏回族自治州', 'jc': '金昌市', 'zy': '张掖市', 'ln': '陇南市', 'jyg': '嘉峪关市', 'ts': '天水市', 'gn': '甘南藏族自治州', 'by': '白银市', 'jq': '酒泉市', 'qy': '庆阳市', 'pl': '平凉市', 'dx': '定西市', 'ww': '武威市', 'sz': '省直部门', 'xq': '兰州新区', 'lzxq': '兰州新区', '白银': '白银市', '定西': '定西市', '酒泉': '酒泉市', '嘉峪关': '嘉峪关市', '陇南': '陇南市', '临夏': '临夏回族自治州', '平凉': '平凉市', '庆阳': '庆阳市', '天水': '天水市', '武威': '武威市', '新区': '兰州新区', '兰州': '兰州市', '张掖': '张掖市', '甘南': '甘南藏族自治州', '省直': '省直部门', '金昌': '金昌市', } dirCs = os.listdir(path) cs = ['微博id', '微博正文', '头条文章url', '原始图片url', '被转发微博原始图片url', '是否为原创微博', '微博视频url', '发布位置', 'date', '发布工具', '点赞数', '转发数', '评论数', 'weiboID', 'weiboName', '市州'] dfWB = pd.DataFrame(columns=cs) cityCount = 0 for dirC in dirCs: if dirC[:1] == '.': continue if not os.path.isdir(os.path.join(path, dirC)): continue if 'weixin' in dirC.lower(): continue if 'tt' in dirC.lower(): continue if not cityShorten[dirC] in cities: continue print(' city: ', cityShorten[dirC], dirC) cityCount += 1 # City LN cols = ['微博id', '微博正文', '头条文章url', '原始图片url', '被转发微博原始图片url', '是否为原创微博', '微博视频url', '发布位置', 'date', '发布工具', '点赞数', '转发数', '评论数'] #WB下载工具中的格式 dfWBC = pd.DataFrame(columns=cols) dirCTs = os.listdir(os.path.join(path, dirC)) for dirCT in dirCTs: if dirCT[:1] == '.': continue # 时段 weibo weibo_1 if not os.path.isdir(os.path.join(path, dirC, dirCT)): continue if 'weixin' in dirCT.lower(): continue if 'tt' in dirCT.lower(): continue print(' read WB... dir:',dirCT) dirAs = os.listdir(os.path.join(path, dirC, dirCT)) for dirA in dirAs: if dirA[:1] == '.': continue # 都是账号名称目录下再存账号ID.txt， if not os.path.isdir(os.path.join(path, dirC, dirCT, dirA)): continue ##print('---',dirA) # 账号名称 wbName = dirA fileAs = os.listdir(os.path.join(path, dirC, dirCT, dirA)) if len(fileAs) > 0 and os.path.splitext(fileAs[0])[-1] == '.csv': wbId = fileAs[0][:-4] if len(fileAs) > 1 and wbId.startswith('.'): wbId = fileAs[1][:-4] # 读取文件 ##print('----',wbName, wbId) filename = os.path.join(path, dirC, dirCT, dirA, fileAs[0]) dfdfwb = pd.read_csv(filename, sep=',', header=None, names=cols, index_col=None)#, engine='python', encoding='gbk'#utf-8 dfdfwb = dfdfwb[1:] dfdfwb["weiboID"] = wbId dfdfwb["weiboName"] = wbName dfWBC = dfWBC.append(dfdfwb) #print(wbName, wbId, fileAs[0], dfdfwb.shape, dfWBC.shape) if len(fileAs)>1: print(" +=+= ", fileAs) print(' ', dfWBC.shape) #dfWBC.to_excel("D:/Projects/POM/2021年6月/二季度/全文/WB/WB_"+dirC+".xlsx") dfWBC['市州'] = cityShorten[dirC] dfWB = dfWB.append(dfWBC) print('Read WB finished. cities', cityCount, '; lines', dfWB.shape) #dfWB.to_excel("D:/Projects/POM/2021年7月/2021年上半年/WB_ALL.xlsx") return dfWB # 从数据目录中读取xlsx文件，拼接到一起 def getWXData(path, cities, hasBody=False): # cityShorten cityShorten = {'LZ':'兰州市', 'LX':'临夏回族自治州', 'JC':'金昌市', 'ZY':'张掖市', 'LN':'陇南市', 'JYG':'嘉峪关市', 'TS':'天水市', 'GN':'甘南藏族自治州', 'BY':'白银市', 'JQ':'酒泉市', 'QY':'庆阳市', 'PL':'平凉市', 'DX':'定西市', 'WW':'武威市', 'SZ':'省直部门', 'XQ': '兰州新区', 'LZXQ': '兰州新区', 'lz': '兰州市', 'lx': '临夏回族自治州', 'jc': '金昌市', 'zy': '张掖市', 'ln': '陇南市', 'jyg': '嘉峪关市', 'ts': '天水市', 'gn': '甘南藏族自治州', 'by': '白银市', 'jq': '酒泉市', 'qy': '庆阳市', 'pl': '平凉市', 'dx': '定西市', 'ww': '武威市', 'sz': '省直部门', 'xq': '兰州新区', 'lzxq': '兰州新区', '白银': '白银市', '定西': '定西市', '酒泉': '酒泉市', '嘉峪关': '嘉峪关市', '陇南': '陇南市', '临夏': '临夏回族自治州', '平凉': '平凉市', '庆阳': '庆阳市', '天水': '天水市', '武威': '武威市', '新区': '兰州新区', '兰州': '兰州市', '张掖': '张掖市', '甘南': '甘南藏族自治州', '省直': '省直部门', '金昌': '金昌市', } dirBatches = os.listdir(path) cols = ['公众号', '链接', '日期', '标题', '内容', '头条', '市州', '阅读数'] dfWX = pd.DataFrame(columns=cols) countC = 0 countFnC = 0 # 监测批次目录 for dirBatch in dirBatches: if not os.path.isdir(os.path.join(path, dirBatch)): continue # 仅目录 # City LN # 列出市州文件名称 fileCs = os.listdir(os.path.join(path, dirBatch)) count = 0 for fileC in fileCs: if fileC[:1] == '.': continue # 处理目录 if os.path.isdir(os.path.join(path, dirBatch, fileC)) and 'weixin' in fileC.lower(): print(' ', os.path.join(path, dirBatch, fileC)) fs = os.listdir(os.path.join(path, dirBatch, fileC)) for f in fs: fe = os.path.splitext(f)[-1] if fe == '.xlsx' or fe == '.xls': fName = os.path.splitext(fileC)[0] cityname = cityShorten[dirBatch] if cityname in cities: dfdfwxc = pd.read_excel(os.path.join(path, dirBatch, fileC, f)) dfdfwxc['市州'] = cityname print(' read wx: ', cityShorten[dirBatch], dirBatch, fName, dfdfwxc.shape) dfWX = dfWX.append(dfdfwxc) count = count + 1 # 处理文件 fExt = os.path.splitext(fileC)[-1] if fExt != '.xlsx' and fExt != '.xls': continue # 限制文件扩展名 fName = os.path.splitext(fileC)[0] cityname = cityShorten[dirBatch] if cityname in cities: dfdfwxc = pd.read_excel(os.path.join(path, dirBatch, fileC)) dfdfwxc['市州'] = cityShorten[dirBatch] print(' read wx: ', cityShorten[dirBatch], dirBatch, fName, dfdfwxc.shape) dfWX = dfWX.append(dfdfwxc) count = count + 1 countFnC += count if count > 0: countC += 1 print(" Read WX Finished. cities ", countC, '; Files', countFnC, '; lines ', dfWX.shape[0]) #dfWX.to_excel("D:/Projects/POM/2021年7月/2021年上半年/WX_ALL.xlsx") return dfWX # 从数据目录中读取xlsx文件，拼接到一起 def getTTData(path, cities, hasBody=False): # cityShorten cityShorten = {'LZ':'兰州市', 'LX':'临夏回族自治州', 'JC':'金昌市', 'ZY':'张掖市', 'LN':'陇南市', 'JYG':'嘉峪关市', 'TS':'天水市', 'GN':'甘南藏族自治州', 'BY':'白银市', 'JQ':'酒泉市', 'QY':'庆阳市', 'PL':'平凉市', 'DX':'定西市', 'WW':'武威市', 'SZ':'省直部门', 'XQ': '兰州新区', 'LZXQ': '兰州新区', 'lz': '兰州市', 'lx': '临夏回族自治州', 'jc': '金昌市', 'zy': '张掖市', 'ln': '陇南市', 'jyg': '嘉峪关市', 'ts': '天水市', 'gn': '甘南藏族自治州', 'by': '白银市', 'jq': '酒泉市', 'qy': '庆阳市', 'pl': '平凉市', 'dx': '定西市', 'ww': '武威市', 'sz': '省直部门', 'xq': '兰州新区', 'lzxq': '兰州新区', '白银': '白银市', '定西': '定西市', '酒泉': '酒泉市', '嘉峪关': '嘉峪关市', '陇南': '陇南市', '临夏': '临夏回族自治州', '平凉': '平凉市', '庆阳': '庆阳市', '天水': '天水市', '武威': '武威市', '新区': '兰州新区', '兰州': '兰州市', '张掖': '张掖市', '甘南': '甘南藏族自治州', '省直': '省直部门', '金昌': '金昌市', } dirCs = os.listdir(path) #account date title nread ncomment content url origin cs = ['account', 'date', 'title', 'nread', 'ncomment', 'content', 'url', 'origin', 'city'] dfTT = pd.DataFrame(columns=cs) cityCount = 0 for dirC in dirCs: if dirC[:1] == '.': continue if not os.path.isdir(os.path.join(path, dirC)): continue if 'weixin' in dirC.lower(): continue if 'weibo' in dirC.lower(): continue if not cityShorten[dirC] in cities: continue print(' city: ', cityShorten[dirC], dirC) cityCount += 1 # City LN dfTTC = pd.DataFrame(columns=cs) dirCTs = os.listdir(os.path.join(path, dirC)) for dirCT in dirCTs: if dirCT[:1] == '.': continue # 时段 weibo weibo_1 if not os.path.isdir(os.path.join(path, dirC, dirCT)): continue if 'weixin' in dirCT.lower(): continue if 'weibo' in dirCT.lower(): continue if 'tt' in dirCT.lower(): print(' read TT... dir:',dirCT) fns = os.listdir(os.path.join(path, dirC, dirCT)) for fn in fns: if fn[:1] == '.': continue if not fn[-5:] == '.xlsx': continue #print('---',fn) # 账号名称 ttName = fn[fn.index('_')+1:] ttName = ttName[:ttName.index('_')] #D:\Projects\POM\DATA\2022年11月\10月报告\全文\LN\TT fileAs = os.path.join(path, dirC, dirCT, fn) #print(' ', ttName, fileAs) if len(fileAs) > 0: try: dfdftt = pd.read_excel(fileAs) dfTTC = dfTTC.append(dfdftt) except: print("read file failed. ", fileAs) #dfdfwb = pd.read_csv(filename, sep=',', header=None, names=cols, # index_col=None)#, engine='python', encoding='gbk'#utf-8 #dfdfwb = dfdfwb[1:] #dfdfwb["weiboID"] = wbId #dfdfwb["weiboName"] = wbName #dfTTC = dfTTC.append(dfdfwb) #print(ttName, '读入：', dfdftt.shape[0], ' 总计：', dfTTC.shape[0]) #if len(fileAs)>1: # print(" +=+= ", fileAs) print(' 读入头条数据行数', dfTTC.shape) #dfTTC.to_excel("D:/Projects/POM/2021年6月/二季度/全文/WB/WB_"+dirC+".xlsx") dfTTC['city'] = cityShorten[dirC] dfTT = dfTT.append(dfTTC) print('Read TT finished. cities', cityCount, '; lines', dfTT.shape) #dfTT.to_excel("D:/Projects/POM/2021年7月/2021年上半年/WB_ALL.xlsx") return dfTT def fetch_chinese(s): pattern =re.compile(r'[^\u4e00-\u9fa5]') sc = re.sub(pattern, '', s) return sc def doWBData(): dfAccount = pd.read_excel('D:/Projects/POM/DATA/2023年7月/7.11陇南转发/全国报送系统表单_2023.6.30.xlsx') dfAccount = dfAccount[dfAccount['账号类型']=='新浪微博'] dfAccount['微信biz/oid/账号ID'] = dfAccount['微信biz/oid/账号ID'].astype('int64') dfwb1 = pd.read_csv('D:/Projects/POM/DATA/2023年7月/7.11陇南转发/weibo_2/weibo1.csv', sep=',',index_col=None)#, engine='python', encoding='gbk'#utf-8 dfwb1 = dfwb1.fillna(0) dfwb1['user_id'] = dfwb1['user_id'].astype('int64') dfwb1.rename(columns={'id':'微博id', 'content':'微博正文', 'article_url':'头条文章url', 'original_pictures':'原始图片url', 'retweet_pictures':'被转发微博原始图片url', 'original':'是否为原创微博', 'video_url':'微博视频url', 'publish_place':'发布位置', 'publish_time':'发布时间', 'publish_tool':'发布工具', 'up_num':'点赞数', 'retweet_num':'转发数', 'comment_num':'评论数'}, inplace = True) print(dfAccount.shape) print(dfwb1.shape, dfwb1.dtypes) if 1: sDir = 'D:/Projects/POM/DATA/2023年7月/7.11陇南转发/全文/LN/weibo_3/' i=0 j=0 for uid in dfwb1['user_id'].unique(): dfa1 = dfAccount[dfAccount['微信biz/oid/账号ID']==uid] dfa1.reset_index(inplace=True) if dfa1.shape[0]>0: sA = str(dfa1.loc[0,'账号名称']) #print(dfa1['账号名称']) i = i + 1 dfwba = dfwb1.loc[dfwb1['user_id']==uid] os.mkdir(sDir+sA) #微博id,微博正文,头条文章url,原始图片url,被转发微博原始图片url,是否为原创微博,微博视频url, # 发布位置,发布时间,发布工具,点赞数,转发数,评论数 dfwba = dfwba[['微博id', '微博正文', '头条文章url', '原始图片url', '被转发微博原始图片url', '是否为原创微博', '微博视频url', '微博视频url', '发布位置', '发布时间', '发布工具', '点赞数', '转发数', '评论数']] dfwba = dfwba.reset_index() dfwba.to_csv(sDir+sA+'/'+str(uid)+'.csv', encoding='utf_8_sig', index=0, quoting=1) else: j = j+1 print('found ', i, '; nofound', j) i=0 j=0 if 1: sDir = 'D:/Projects/POM/DATA/2023年7月/7.11陇南转发/全文/LN/weibo_4/' dfwb2 = pd.read_csv('D:/Projects/POM/DATA/2023年7月/7.11陇南转发/weibo_2/weibo2.csv', sep=',',index_col=None)#, engine='python', encoding='gbk'#utf-8 dfwb2 = dfwb2.fillna(0) dfwb2['user_id'] = dfwb2['user_id'].astype('int64') dfwb2.rename(columns={'id':'微博id', 'text':'微博正文', 'article_url':'头条文章url', 'pics':'原始图片url', 'topics':'被转发微博原始图片url','source':'是否为原创微博','video_url':'微博视频url', 'location':'发布位置', 'created_at':'发布时间', 'bid':'发布工具', 'attitudes_count':'点赞数', 'reposts_count':'转发数', 'comments_count':'评论数'}, inplace = True) print(dfwb2.shape) for uid in dfwb2['user_id'].unique(): dfa2 = dfAccount[dfAccount['微信biz/oid/账号ID']==uid] dfa2.reset_index(inplace=True) if dfa2.shape[0]>0: sA = str(dfa2.loc[0, '账号名称']) i = i+1 dfwba = dfwb2.loc[dfwb2['user_id']==uid] os.mkdir(sDir+sA) #微博id,微博正文,头条文章url,原始图片url,被转发微博原始图片url,是否为原创微博,微博视频url,发布位置,发布时间,发布工具,点赞数,转发数,评论数 dfwba = dfwba[['微博id', '微博正文', '头条文章url', '原始图片url', '被转发微博原始图片url', '是否为原创微博', '微博视频url', '发布位置', '发布时间', '发布工具', '点赞数', '转发数', '评论数']] dfwba = dfwba.reset_index() dfwba.to_csv(sDir+sA+'/'+str(uid)+'.csv', encoding='utf_8_sig', index=0, quoting=1) else: #print(uid) j = j+1 print('found ', i, '; nofound', j) if __name__ == "__main__": #doWBData() #exit(0) starttime = datetime.datetime.now() _RATIO = 0.5 isDoWX = True isDoWB = True isDoTT = True cities = [ '临夏回族自治州', '白银市', '定西市', '酒泉市', '嘉峪关市', '平凉市', '庆阳市', '天水市', '武威市', '兰州新区', '陇南市', '兰州市', '张掖市', '甘南藏族自治州', '金昌市', '省直部门', # 共12市2州1新区 ] ''' cities = [ '临夏回族自治州', '白银市', '定西市', '酒泉市', '天水市', '陇南市', #'省直部门', # 共12市2州1新区 ] ''' cities = ['陇南市',] #cities = ['陇南市', '临夏回族自治州', '白银市', '定西市', '酒泉市', '平凉市','武威市','天水市'] #cities = ['陇南市'] # 转发任务 #dfTask = pd.read_excel('D:/Projects/POM/DATA/2022年S2/S2/全省政务新媒体二季度转发信息条目.xls') dfTask = pd.read_excel('D:/Projects/POM/DATA/2023年7月/7.11陇南转发/陇南7月上旬转发台账.xlsx') sTaskTitle = '标题' sTaskDate = '推送时间' # 删除标题列为空的行 dfTask.dropna(axis=0,subset = ["标题"]) yT0 = dfTask.columns.get_loc('序号') yT1 = dfTask.columns.get_loc('标题') # 账号信息 strFnAccount = 'D:/Projects/POM/DATA/2023年7月/7.11陇南转发/全国报送系统表单_2023.6.30.xlsx' dfAllAccount = pd.read_excel(strFnAccount) # 添加列 dfAllAccount.loc[:, '转发数'] = 0 #dfAllAccount.loc[:, '阅读数'] = 0 dfAllAccount = pd.concat([dfAllAccount, pd.DataFrame(np.zeros((dfAllAccount.shape[0], dfTask.shape[0])), columns=dfTask['序号'].astype(str).tolist())], axis=1) # 整理数据 dfAllAccount['市/省局'] = dfAllAccount['市/省局'].fillna('省直部门') dfAllAccount['区县/地方部门'] = dfAllAccount['区县/地方部门'].fillna('市直部门') dfAllAccount.loc[(dfAllAccount['市/省局'].isin(['临夏回族自治州', '甘南藏族自治州'])) & (dfAllAccount['区县/地方部门'] == '市直部门'), '区县/地方部门'] = '州直部门' dfAllAccount.loc[(dfAllAccount['市/省局'].isin(['省直部门'])) & (dfAllAccount['区县/地方部门'] == '市直部门'), '区县/地方部门'] = '省直部门' # 过长名称替换为简称，便于绘图 dfAllAccount.loc[dfAllAccount['区县/地方部门'] == '积石山保安族东乡族撒拉族自治县', '区县/地方部门'] = '积石山县' dfAllAccount.loc[dfAllAccount['区县/地方部门'] == '阿克塞哈萨克族自治县', '区县/地方部门'] = '阿克塞自治县' yAccountName = dfAllAccount.columns.get_loc('账号名称') yAccountCity = dfAllAccount.columns.get_loc('市/省局') yAccountCounty = dfAllAccount.columns.get_loc('区县/地方部门') yAccountUnit = dfAllAccount.columns.get_loc('单位全称') # 省直部门账号部门简称 fnTemplate = 'D:/Projects/POM/DATA/2023年7月/7.11陇南转发/POM_ForewardTemplate.docx' # 数据根目录， strPath = ['D:/Projects/POM/DATA/2023年7月/7.11陇南转发/全文/', ] strOutputPath = 'D:/Projects/POM/DATA/2023年7月/7.11陇南转发/转发/' context = { "year": "2023", "month": "7", "pubMonth": "7", "dateStart": "2023年7月1日", "dateEnd": "2023年7月10日" } ################################################ # 创建存储矩阵 # 按照转发任务创建统计矩阵 colRR = ['市州', '类型', '账号名称', '单位名称', '省直部门', '区县', '转发数', '阅读数'] for ididid in dfTask['序号'][0:dfTask['标题'].count()].tolist(): #for ididid in range(1, dfTask['标题'].count()): colRR.append(str(ididid)) # 用于保存每一条转发任务的账号和文章 dfO = pd.DataFrame(columns=['任务序号', '任务名称', '类型', '公众号', '日期', '内容', '链接', '市州'] ) # WX if isDoWX: print('=============================================================') print('---- WX ----') dfWX = pd.DataFrame() for strP in strPath: ddff = getWXData(strP, cities) dfWX = dfWX.append(ddff) dfWX = dfWX.fillna(value=0) yWXtitle = dfWX.columns.get_loc('标题') yWXnread = dfWX.columns.get_loc('阅读数') yWXdate = dfWX.columns.get_loc('日期') yWXurl = dfWX.columns.get_loc('链接') # 公众号 链接 日期 标题 内容 头条 city ## 逐个市州统计每个账号的转发情况 #cities = dfWX['市州'].unique() for city in cities: print('---- WX title match', city, ' ----' ) # 本市微信数据 dataC = dfWX.loc[dfWX['市州'] == city].copy() # 获取微信账号数 accounts = dataC['公众号'].unique() # 所有微信账号数 maskCWX = ( (dfAllAccount['账号类型'] == '微信服务号')|(dfAllAccount['账号类型'] == '微信订阅号') ) & (dfAllAccount['市/省局'] == city) accountNumCWX = maskCWX.tolist().count(True) # 按获取得微信账号遍历 for account in accounts: #print(account) # 该账号的所有文章 dataA = dataC.loc[dataC['公众号'] == account].copy() # 一个公众号的所有文章 sR = pd.Series(dtype='object') sR['类型'] = '微信' sR['市州'] = city sR['账号名称'] = account count = 0 arn = 0 # 从账号信息中匹配该账号详细信息 mask = ( (dfAllAccount['账号类型'] == '小程序+微信') | (dfAllAccount['账号类型'] == '微信服务号') | (dfAllAccount['账号类型'] == '微信订阅号') ) & (dfAllAccount['市/省局'] == city) & (dfAllAccount['账号名称']==account) if mask.any(): sxq = str(dfAllAccount.loc[mask, '区县/地方部门'].values[0]) if sxq.lower() !='nan': sR['区县'] = sxq sdwmc = str(dfAllAccount.loc[mask, '单位全称'].values[0]) if sdwmc.lower() != 'nan': sR['单位名称'] = sdwmc else: print(' !!!! 微信', account, '在', city, '无详细信息' ) continue # 按任务标题逐个匹配所有发文，得到每篇任务的转发情况 for i in range(dfTask['标题'].count()): # 对于每一篇任务文章 rn = dfTask.iloc[i, yT0] # 序号 ssrt = str(dfTask.iloc[i, yT1]) # 标题/内容 rt = fetch_chinese(ssrt) # 只取汉字 forwarded = 0 # 转发数 readNum = 0 # 阅读数 # 查看该账号的所有文章 for j in range(dataA.shape[0]): str1 = fetch_chinese(str(dataA.iloc[j, yWXtitle])) # 只取汉字 # 任务标题过长，截取前半部分进行对比 if len(rt) > len(str1): strRT = rt[:len(str1)] else:#文章标题过长，只比较任务标题长度部分 strRT = rt str1 = str1[:len(rt)] ratio = difflib.SequenceMatcher(None, strRT, str1).quick_ratio() # 遇到相似的，认为已转发，即跳出不再查找 if ratio > _RATIO: forwarded += 1 readNum += int(dataA.iloc[j, yWXnread]) if forwarded > 0: break sR[str(rn)] = forwarded # 记录该篇文章的转发数 count += forwarded # 累加该篇文章的转发数 arn += readNum # 累加该篇文章的阅读数 # 记录该篇任务转发情况加入 if forwarded > 0: dfO = dfO.append([{'任务序号': rn, '任务名称': ssrt, '类型': '微信', '公众号': account, '日期': dataA.iloc[j, yWXdate], '内容': str1, '链接': dataA.iloc[j, yWXurl], '市州': city, '阅读数': readNum, }], ignore_index=True) #记录该任务的转发情况 dfAllAccount.loc[mask, str(rn)] = forwarded #记录该账号的总转发数 dfAllAccount.loc[mask, '转发数'] = count sR['转发数'] = count sR['阅读数'] = arn # 全市总转发文章篇数 ccwx = dfAllAccount.loc[maskCWX, '转发数'].sum() # 全市总转发率 rcc = ccwx/accountNumCWX/dfTask.shape[0] print(' ', city, '共有', accountNumCWX, '个微信号，获取数据', len(accounts), '个。共转发', ccwx, '次，转发率{:.1f}%'.format(rcc*100) ) #countWxForewards = dfRR.shape[0] #print(' 获取 WX 账号数', len(dfWX['公众号'].unique()),'参与转发账号数', countWxForewards) # WB if isDoWB: print('=============================================================') print('---- WB data read ----') #获取微博数据 dfWB = pd.read_excel('D:/Projects/POM/DATA/2023年7月/7.11陇南转发/微博全文.xlsx') #for strP in strPath: ## ddff = getWBData(strP, cities) # dfWB = dfWB.append(ddff) print('----', dfWB.shape) #=========================================================================================== #=========================================================================================== yWBcontent = dfWB.columns.get_loc('微博正文') yWBdate = dfWB.columns.get_loc('date') yWBurl = dfWB.columns.get_loc('头条文章url') #dfWB.to_excel('D:/Projects/POM/DATA/2023年7月/7.11陇南转发/转发/微博全文.xlsx') ################################################ # WB # 微博id 微博正文 头条文章url 原始图片url 被转发微博原始图片url 是否为原创微博 微博视频url 发布位置 date # 发布工具 点赞数 转发数 评论数 weiboID weiboName city #cities = dfWB['市州'].unique() for city in cities: print('---- WB match', city, ' ----' ) # 本市微博数据 dataC = dfWB.loc[dfWB['市州'] == city].copy() # 获取数据的微博账号 accounts = dataC['weiboName'].unique() # 本市所有微博账号 maskCWB = (dfAllAccount['账号类型'] == '新浪微博') & (dfAllAccount['市/省局'] == city) accountNumCWB = maskCWB.tolist().count(True) # 按获取的微博账号遍历 for account in accounts: # print(account) # 该公众号的所有文章 dataA = dataC.loc[dataC['weiboName'] == account].copy() sR = pd.Series(dtype='object') sR['类型'] = '新浪微博' sR['市州'] = city sR['账号名称'] = account count = 0 # 为转发账号匹配单位全称和所属县区 mask = ( dfAllAccount['账号类型'] == '新浪微博' ) & (dfAllAccount['市/省局'] == city) & (dfAllAccount['账号名称']==account) if mask.any(): sxq = str(dfAllAccount.loc[mask, '区县/地方部门'].values[0]) if sxq.lower() !='nan': sR['区县'] = sxq sdwmc = str(dfAllAccount.loc[mask, '单位全称'].values[0]) if sdwmc.lower() != 'nan': sR['单位名称'] = sdwmc else: print(' !!!! 微博', account, '在', city, '无详细信息' ) continue # 按任务标题逐个匹配所有发文，得到每篇任务的转发情况 for i in range(dfTask['标题'].count()): rn = dfTask.iloc[i, yT0] # 任务序号 ssrt = str(dfTask.iloc[i, yT1]) # 任务标题 rt = fetch_chinese(ssrt) # 只取中文 forwarded = 0 # 对该账号的所有文章 for j in range(dataA.shape[0]): str0 = str(dataA.iloc[j, yWBcontent]) str1 = fetch_chinese(str0) str2 = str1[:len(rt)] # 取任务标题相同汉字数进行比较 ratio = difflib.SequenceMatcher(None, rt, str2).quick_ratio() if ratio > _RATIO: forwarded += 1 if forwarded > 0: break #记记录该任务的转发情况 dfAllAccount.loc[mask, str(rn)] = forwarded sR[str(rn)] = forwarded # 转发数累加到本账号里 count += forwarded # 记录该篇任务转发情况加入 if forwarded > 0: dfO = dfO.append([{'任务序号': rn, '任务名称': rt, '类型': '新浪微博', '公众号': account, '日期': dataA.iloc[j, yWBdate], '内容': str1, '链接': dataA.iloc[j, yWBurl], '市州': city, }], ignore_index=True) # 记录该账号的总转发数 dfAllAccount.loc[mask, '转发数'] = count sR['转发数'] = count # 全市总转发文章篇数 ccwb = dfAllAccount.loc[maskCWB, '转发数'].sum() # 全市总转发率 rcc = ccwb/accountNumCWB/dfTask.shape[0] print(' ', city, '共有', accountNumCWB, '个微博号，获取数据', len(accounts), '个。共转发', ccwb, '次，转发率{:.1f}%'.format(rcc*100) ) #countWbForewards = dfRR.shape[0] - countWxForewards #print(' 获取 WB 账号数', len(dfWB['weiboName'].unique()), '参与转发账号数', countWbForewards) # TT if isDoTT: print('=============================================================') print('---- TT data read ----') # id userId source city tid cellType title # time-stamp date url commentCount readNum likeNum showNum # 获取头条数据 dfTT = pd.DataFrame() for strP in strPath: ddff = getTTData(strP, cities) dfTT = dfTT.append(ddff) yTTtitle = dfTT.columns.get_loc('title') yTTdate = dfTT.columns.get_loc('date') yTTurl = dfTT.columns.get_loc('url') # 逐个市州统计账号转发情况 for city in cities: print("++++++++++++++++++++++++++++++++++++++++++++++++++") print('---- TT title match', city, ' ----' ) # 本市头条数据 dataC = dfTT.loc[dfTT['city'] == city].copy() # 获取数据的头条账号 accounts = dataC['account'].unique() # 本市所有头条账号信息 maskCTT = (dfAllAccount['账号类型'] == '今日头条') & (dfAllAccount['市/省局'] == city) accountNumCTT = maskCTT.tolist().count(True) # 按头条数据的账号遍历 for account in accounts: #print(account) # 该账号的所有文章 dataA = dataC[dataC['account']==account] sR = pd.Series([], dtype=pd.StringDtype()) sR['类型'] = '今日头条' sR['市州'] = city sR['账号名称'] = account count = 0 # 为转发账号匹配单位全称和所属县区 mask = ( dfAllAccount['账号类型'] == '今日头条' ) & (dfAllAccount['市/省局'] == city) & (dfAllAccount['账号名称']==account) if mask.any(): sxq = str(dfAllAccount.loc[mask, '区县/地方部门'].values[0]) if sxq.lower() !='nan': sR['区县'] = sxq sdwmc = str(dfAllAccount.loc[mask, '单位全称'].values[0]) if sdwmc.lower() != 'nan': sR['单位名称'] = sdwmc else: print(' !!!! 头条', account, '在', city, '无详细信息' ) continue # 按任务标题逐个匹配所有发文，得到每篇任务的转发情况 for i in range(dfTask['标题'].count()): # 对于每一篇任务文章 rn = dfTask.iloc[i, yT0] # 任务序号 ssrt = str(dfTask.iloc[i, yT1]) # 任务标题 rt = fetch_chinese(ssrt) # 只取中文 forwarded = 0 # 查看该账号的所有文章 for j in range(dataA.shape[0]): str0 = str(dataA.iloc[j, yTTtitle]) str1 = fetch_chinese(str0) # if len(rt) > len(str1): # 若任务标题过长，截取前半部分进行对比 strRT = rt[:len(str1)] else: #若文章标题过长，只比较任务标题长度部分 strRT = rt str1 = str1[:len(rt)] ratio = difflib.SequenceMatcher(None, strRT, str1).quick_ratio() if ratio > _RATIO: forwarded += 1 if forwarded > 0: break #记录该任务转发情况 dfAllAccount.loc[mask, str(rn)] = forwarded sR[str(rn)] = forwarded count += forwarded if forwarded > 0: dfO = dfO.append([{'任务序号': rn, '任务名称': rt, '类型': '今日头条', '公众号': account, '日期': dataA.iloc[j, yTTdate], '内容': str1, '链接': dataA.iloc[j, yTTurl], '市州': city, }], ignore_index=True) # 记录该账号转发情况 dfAllAccount.loc[mask, '转发数'] = count sR['转发数'] = count # 全市总转发文章篇数 cctt = dfAllAccount.loc[maskCTT, '转发数'].sum() # 全市总转发率 rcc = cctt/accountNumCTT/dfTask.shape[0] print(' ', city, '共有', accountNumCTT, '个头条号，获取数据', len(accounts), '个。共转发', cctt, '次，转发率{:.1f}%'.format(rcc*100) ) #countTtForewards = dfRR.shape[0] - countWxForewards - countWbForewards #print(' 获取 TT 账号数', len(dfTT['account'].unique()),'参与转发账号数', countTtForewards) if isDoWX or isDoWB or isDoTT: print('=============================================================') print('---- STATISTICS ----') print('=============================================================') dfAllAccount.to_excel(strOutputPath + '甘肃省_转发账号.xlsx') dfO.to_excel(strOutputPath + '甘肃省_转发文章.xlsx') print('---- 统计市州转发率 ----') for city in cities: #if city in ['兰州新区', '省直部门']: # continue print(" add up city", city) maskC = ( (dfAllAccount['账号类型'] == '新浪微博') | (dfAllAccount['账号类型'] == '微信服务号') | (dfAllAccount['账号类型'] == '微信订阅号') | (dfAllAccount['账号类型'] == '今日头条') ) & (dfAllAccount['市/省局'] == city) # dfdfC = dfAllAccount.loc[((dfAllAccount['账号类型'] == '新浪微博') # | (dfAllAccount['账号类型'] == '微信服务号') # | (dfAllAccount['账号类型'] == '微信订阅号') # | (dfAllAccount['账号类型'] == '今日头条')) # & (dfAllAccount['市/省局'] == city)].copy() dfdfC = dfAllAccount.loc[maskC,:] dfdfC.to_excel(strOutputPath + city + '_转发账号.xlsx') dfOCity = dfO[dfO['市州'] == city] dfO.to_excel(strOutputPath + city + '_转发文章.xlsx') #dfRRCity = dfRR.loc[dfRR['市州'] == city].copy() ######################################################################################################### # 统计市/州直部门转发数 dfdfCD = dfdfC.loc[dfdfC['区县/地方部门'].isin(['州直部门', '市直部门', '省直部门'])].copy() dfdfCDA = pd.pivot_table(dfdfCD, index=['单位全称'], values=['账号名称'], aggfunc=['count'], fill_value='', margins=True, margins_name='总计') dfdfCDC = pd.pivot_table(dfdfCD, index=['单位全称'], values=['转发数'], aggfunc=['sum'], fill_value='', margins=True, margins_name='总计') #dfdfCDR = pd.pivot_table(dfdfCD, index=['单位全称'], values=['阅读数'], # aggfunc=['sum'], fill_value='', margins=True, margins_name='总计') dfdfCD_A = pd.concat([dfdfCDA, dfdfCDC], axis=1) #print('-', dfdfCD_A.columns.values) # 合并多层索引MultiIndex dfdfCD_A.columns = ['_'.join(col) for col in dfdfCD_A.columns.values] #print('=', dfdfCD_A.columns.values) # 计算转发率 dfdfCD_A['rate'] = dfdfCD_A.apply( lambda x: int(x['sum_转发数'] / x['count_账号名称'] / dfTask['标题'].count() * 1000) / 1000.0, axis=1) # 排序 dfdfCD_AD = dfdfCD_A[0:dfdfCD_A.shape[0] - 1].sort_values(by='rate', ascending=False) dfdfCD_AD = pd.concat([dfdfCD_AD, dfdfCD_A[dfdfCD_A.shape[0] - 1:dfdfCD_A.shape[0]]], axis=0) dfdfCD_AD.to_excel(strOutputPath + city + '部门转发统计表.xlsx') #dfDD ########################################################################################## # 全市/州账号按'区县'统计 # 发现目前版本pivot_table函数aggfunc用列表时，前几列计算值不准确 # 所以，暂时单列计算，再合并 #dfdfCD = dfdfC.loc[dfdfC['区县/地方部门'].isin(['州直部门', '市直部门', '省直部门'])].copy() dfdfCA = pd.pivot_table(dfdfC, index=['区县/地方部门'], values=['账号名称'], aggfunc=['count'], fill_value='', margins=True, margins_name='总计') dfdfCC = pd.pivot_table(dfdfC, index=['区县/地方部门'], values=['转发数'], aggfunc=['sum'], fill_value='', margins=True, margins_name='总计') #dfdfCDR = pd.pivot_table(dfdfCD, index=['单位全称'], values=['阅读数'], # aggfunc=['sum'], fill_value='', margins=True, margins_name='总计') dfdfC_A = pd.concat([dfdfCA, dfdfCC], axis=1) #print('-', dfdfCD_A.columns.values) # 合并多层索引MultiIndex dfdfC_A.columns = ['_'.join(col) for col in dfdfC_A.columns.values] #print('=', dfdfCD_A.columns.values) # 计算转发率 dfdfC_A['rate'] = dfdfC_A.apply( lambda x: int(x['sum_转发数'] / x['count_账号名称'] / dfTask['标题'].count() * 1000) / 1000.0, axis=1) # 排序 dfdfC_AD = dfdfC_A[0:dfdfC_A.shape[0] - 1].sort_values(by='rate', ascending=False) dfdfC_AD = pd.concat([dfdfC_AD, dfdfC_A[dfdfC_A.shape[0] - 1:dfdfC_A.shape[0]]], axis=0) dfdfC_AD.to_excel(strOutputPath + city + '转发统计表.xlsx') #dfCC ######################################################### # # 生成报告 tpl = DocxTemplate(fnTemplate) if city in ['临夏回族自治州', '甘南藏族自治州']: sL0 = '州' else: sL0 = '市' info = { "strL0":sL0, "strL1":"区县", "taskCount": dfTask['标题'].count(), "aNum": int(dfdfC_AD.iloc[-1]['count_账号名称']), "fNum": int(dfdfC_AD.iloc[-1]['sum_转发数']), "r": '%.1f'%(dfdfC_AD.iloc[-1]['rate']*100.0), # "dNum": int(dfdfCD_AD.iloc[-1]['count_账号名称']), # 部门总账号数 "dFNum": int(dfdfCD_AD.iloc[-1]['sum_转发数']), # 部门总转发数 "dr": '%.1f'%(dfdfCD_AD.iloc[-1]['rate']*100.0), # 部门平均转发率 } context.update(info) # 县区转发率表格 t1_list = [] for index, row in dfdfC_AD.iterrows(): if index == "总计": continue t1_a = {'county': str(index), 'rate': '%.1f'%(row['rate']*100.0), 'account': int(row['count_账号名称']), 'fNum': int(row['sum_转发数']) } t1_list.append(t1_a) context['t1_contents'] = t1_list # 部门转发率表格 t2_list = [] for index, row in dfdfCD_AD.iterrows(): if index == "总计": continue t2_a = {'name': str(index), 'rate': '%.1f'%(row['rate']*100.0), 'account': int(row['count_账号名称']), 'fNum': int(row['sum_转发数']) } t2_list.append(t2_a) context['t2_contents'] = t2_list # 转发任务列表 t3_list = [] for index, row in dfTask.iterrows(): t3_a = {'id': row['序号'], 'title': row['标题'], 'date': ts2date(row[sTaskDate], '%m月%d日') } t3_list.append(t3_a) context['t3_contents'] = t3_list # 绘制区县转发率图 drawBar(dfdfC_AD['rate'][:-1], dfdfC_AD.index[:-1], '区县转发率', os.path.join(strOutputPath, '_' + city + '_graphCounty.png')) dc = { 'graphCounty': InlineImage(tpl, os.path.join(strOutputPath, '_' + city+'_graphCounty.png'), width=Mm(120)), } context.update(dc) tpl.render(context) tpl.save(strOutputPath+city+'转发统计报告_2023年{}月份.docx'.format(context['month'])) endtime = datetime.datetime.now() usedtime = endtime - starttime print("time: ", usedtime)