import datetime import csv import pandas as pd import numpy as np import glob, os, re, time import matplotlib.pyplot as plt from matplotlib.ticker import FuncFormatter from difflib import SequenceMatcher from collections import Counter import difflib from docxtpl import DocxTemplate from docxtpl import InlineImage from docx.shared import Mm import jieba import jieba.posseg as pseg def fetch_chinese(s): pattern =re.compile(r'[^\u4e00-\u9fa5]') sc = re.sub(pattern, '', s) return sc def toDate(strDT): dt = pd.to_datetime(strDT, errors='coerce') dts = '' # print('-+-+:', type(dt), dt) if not pd.isna(dt): dts = dt.strftime('%m-%d') return dts # 画柱状图 def drawBar(data, recipe, title='', fn=''): plt.figure(figsize=(6, 4)) plt.rcParams['font.sans-serif'] = ['SimHei'] plt.rcParams['axes.unicode_minus'] = False counties = recipe countyRates = data plt.bar(counties, countyRates, width=0.5) plt.xticks(counties, counties, rotation=35) plt.ylim((0, 1)) def to_percent(temp, position): return '%2.0f' % (100 * temp) + '%' plt.gca().yaxis.set_major_formatter(FuncFormatter(to_percent)) plt.title(title, fontsize=16) plt.tight_layout() plt.savefig(fn) # plt.show() plt.cla() plt.clf() plt.close() def getWBData(path, cities, hasBody=False): # cityShorten cityShorten = {'LZ':'兰州市', 'LX':'临夏回族自治州', 'JC':'金昌市', 'ZY':'张掖市', 'LN':'陇南市', 'JYG':'嘉峪关市', 'TS':'天水市', 'GN':'甘南藏族自治州', 'BY':'白银市', 'JQ':'酒泉市', 'QY':'庆阳市', 'PL':'平凉市', 'DX':'定西市', 'WW':'武威市', 'SZ':'省直部门', 'XQ': '兰州新区', 'LZXQ': '兰州新区', 'lz': '兰州市', 'lx': '临夏回族自治州', 'jc': '金昌市', 'zy': '张掖市', 'ln': '陇南市', 'jyg': '嘉峪关市', 'ts': '天水市', 'gn': '甘南藏族自治州', 'by': '白银市', 'jq': '酒泉市', 'qy': '庆阳市', 'pl': '平凉市', 'dx': '定西市', 'ww': '武威市', 'sz': '省直部门', 'xq': '兰州新区', 'lzxq': '兰州新区', '白银': '白银市', '定西': '定西市', '酒泉': '酒泉市', '嘉峪关': '嘉峪关市', '陇南': '陇南市', '临夏': '临夏回族自治州', '平凉': '平凉市', '庆阳': '庆阳市', '天水': '天水市', '武威': '武威市', '新区': '兰州新区', '兰州': '兰州市', '张掖': '张掖市', '甘南': '甘南藏族自治州', '省直': '省直部门', '金昌': '金昌市', } dirCs = os.listdir(path) cs = ['微博id', '微博正文', '头条文章url', '原始图片url', '被转发微博原始图片url', '是否为原创微博', '微博视频url', '发布位置', 'date', '发布工具', '点赞数', '转发数', '评论数', 'weiboID', 'weiboName', '市州'] dfWB = pd.DataFrame(columns=cs) cityCount = 0 for dirC in dirCs: if dirC[:1] == '.': continue if not os.path.isdir(os.path.join(path, dirC)): continue if 'weixin' in dirC.lower(): continue if 'tt' in dirC.lower(): continue if not cityShorten[dirC] in cities: continue print(' city: ', cityShorten[dirC], dirC) cityCount += 1 # City LN cols = ['微博id', '微博正文', '头条文章url', '原始图片url', '被转发微博原始图片url', '是否为原创微博', '微博视频url', '发布位置', 'date', '发布工具', '点赞数', '转发数', '评论数'] #WB下载工具中的格式 dfWBC = pd.DataFrame(columns=cols) dirCTs = os.listdir(os.path.join(path, dirC)) for dirCT in dirCTs: if dirCT[:1] == '.': continue # 时段 weibo weibo_1 if not os.path.isdir(os.path.join(path, dirC, dirCT)): continue if 'weixin' in dirCT.lower(): continue if 'tt' in dirCT.lower(): continue print(' read WB... dir:',dirCT) dirAs = os.listdir(os.path.join(path, dirC, dirCT)) for dirA in dirAs: if dirA[:1] == '.': continue # 都是账号名称目录下再存账号ID.txt， if not os.path.isdir(os.path.join(path, dirC, dirCT, dirA)): continue ##print('---',dirA) # 账号名称 wbName = dirA fileAs = os.listdir(os.path.join(path, dirC, dirCT, dirA)) if len(fileAs) > 0 and os.path.splitext(fileAs[0])[-1] == '.csv': wbId = fileAs[0][:-4] if len(fileAs) > 1 and wbId.startswith('.'): wbId = fileAs[1][:-4] # 读取文件 ##print('----',wbName, wbId) filename = os.path.join(path, dirC, dirCT, dirA, fileAs[0]) dfdfwb = pd.read_csv(filename, sep=',', header=None, names=cols, index_col=None)#, engine='python', encoding='gbk'#utf-8 dfdfwb = dfdfwb[1:] dfdfwb["weiboID"] = wbId dfdfwb["weiboName"] = wbName dfWBC = dfWBC.append(dfdfwb) #print(wbName, wbId, fileAs[0], dfdfwb.shape, dfWBC.shape) if len(fileAs)>1: print(" +=+= ", fileAs) print(' ', dfWBC.shape) #dfWBC.to_excel("D:/Projects/POM/2021年6月/二季度/全文/WB/WB_"+dirC+".xlsx") dfWBC['市州'] = cityShorten[dirC] dfWB = dfWB.append(dfWBC) print('Read WB finished. cities', cityCount, '; lines', dfWB.shape) #dfWB.to_excel("D:/Projects/POM/2021年7月/2021年上半年/WB_ALL.xlsx") return dfWB # 从数据目录中读取xlsx文件，拼接到一起 def getWXData(path, cities, hasBody=False): # cityShorten cityShorten = {'LZ':'兰州市', 'LX':'临夏回族自治州', 'JC':'金昌市', 'ZY':'张掖市', 'LN':'陇南市', 'JYG':'嘉峪关市', 'TS':'天水市', 'GN':'甘南藏族自治州', 'BY':'白银市', 'JQ':'酒泉市', 'QY':'庆阳市', 'PL':'平凉市', 'DX':'定西市', 'WW':'武威市', 'SZ':'省直部门', 'XQ': '兰州新区', 'LZXQ': '兰州新区', 'lz': '兰州市', 'lx': '临夏回族自治州', 'jc': '金昌市', 'zy': '张掖市', 'ln': '陇南市', 'jyg': '嘉峪关市', 'ts': '天水市', 'gn': '甘南藏族自治州', 'by': '白银市', 'jq': '酒泉市', 'qy': '庆阳市', 'pl': '平凉市', 'dx': '定西市', 'ww': '武威市', 'sz': '省直部门', 'xq': '兰州新区', 'lzxq': '兰州新区', '白银': '白银市', '定西': '定西市', '酒泉': '酒泉市', '嘉峪关': '嘉峪关市', '陇南': '陇南市', '临夏': '临夏回族自治州', '平凉': '平凉市', '庆阳': '庆阳市', '天水': '天水市', '武威': '武威市', '新区': '兰州新区', '兰州': '兰州市', '张掖': '张掖市', '甘南': '甘南藏族自治州', '省直': '省直部门', '金昌': '金昌市', } dirBatches = os.listdir(path) cols = ['公众号', '链接', '日期', '标题', '内容', '头条', '市州', '阅读数'] dfWX = pd.DataFrame(columns=cols) countC = 0 countFnC = 0 # 监测批次目录 for dirBatch in dirBatches: if not os.path.isdir(os.path.join(path, dirBatch)): continue # 仅目录 # City LN # 列出市州文件名称 fileCs = os.listdir(os.path.join(path, dirBatch)) count = 0 for fileC in fileCs: if fileC[:1] == '.': continue # 处理目录 if os.path.isdir(os.path.join(path, dirBatch, fileC)) and 'weixin' in fileC: print(' ', os.path.join(path, dirBatch, fileC)) fs = os.listdir(os.path.join(path, dirBatch, fileC)) for f in fs: fe = os.path.splitext(f)[-1] if fe == '.xlsx' or fe == '.xls': fName = os.path.splitext(fileC)[0] cityname = cityShorten[dirBatch] if cityname in cities: dfdfwxc = pd.read_excel(os.path.join(path, dirBatch, fileC, f)) dfdfwxc['市州'] = cityname print(' read wx: ', cityShorten[dirBatch], dirBatch, fName, dfdfwxc.shape) dfWX = dfWX.append(dfdfwxc) count = count + 1 # 处理文件 fExt = os.path.splitext(fileC)[-1] if fExt != '.xlsx' and fExt != '.xls': continue # 限制文件扩展名 fName = os.path.splitext(fileC)[0] cityname = cityShorten[dirBatch] if cityname in cities: dfdfwxc = pd.read_excel(os.path.join(path, dirBatch, fileC)) dfdfwxc['市州'] = cityShorten[dirBatch] print(' read wx: ', cityShorten[dirBatch], dirBatch, fName, dfdfwxc.shape) dfWX = dfWX.append(dfdfwxc) count = count + 1 countFnC += count if count > 0: countC += 1 print("Read WX Finished. cities ", countC, '; Files', countFnC, '; lines ', dfWX.shape[0]) #dfWX.to_excel("D:/Projects/POM/2021年7月/2021年上半年/WX_ALL.xlsx") return dfWX # 从数据目录中读取xlsx文件，拼接到一起 def getTTData(path, cities, hasBody=False): # cityShorten cityShorten = {'LZ':'兰州市', 'LX':'临夏回族自治州', 'JC':'金昌市', 'ZY':'张掖市', 'LN':'陇南市', 'JYG':'嘉峪关市', 'TS':'天水市', 'GN':'甘南藏族自治州', 'BY':'白银市', 'JQ':'酒泉市', 'QY':'庆阳市', 'PL':'平凉市', 'DX':'定西市', 'WW':'武威市', 'SZ':'省直部门', 'XQ': '兰州新区', 'LZXQ': '兰州新区', 'lz': '兰州市', 'lx': '临夏回族自治州', 'jc': '金昌市', 'zy': '张掖市', 'ln': '陇南市', 'jyg': '嘉峪关市', 'ts': '天水市', 'gn': '甘南藏族自治州', 'by': '白银市', 'jq': '酒泉市', 'qy': '庆阳市', 'pl': '平凉市', 'dx': '定西市', 'ww': '武威市', 'sz': '省直部门', 'xq': '兰州新区', 'lzxq': '兰州新区', '白银': '白银市', '定西': '定西市', '酒泉': '酒泉市', '嘉峪关': '嘉峪关市', '陇南': '陇南市', '临夏': '临夏回族自治州', '平凉': '平凉市', '庆阳': '庆阳市', '天水': '天水市', '武威': '武威市', '新区': '兰州新区', '兰州': '兰州市', '张掖': '张掖市', '甘南': '甘南藏族自治州', '省直': '省直部门', '金昌': '金昌市', } dirCs = os.listdir(path) #account date title nread ncomment content url origin cs = ['account', 'date', 'title', 'nread', 'ncomment', 'content', 'url', 'origin', 'city'] dfTT = pd.DataFrame(columns=cs) cityCount = 0 for dirC in dirCs: if dirC[:1] == '.': continue if not os.path.isdir(os.path.join(path, dirC)): continue if 'weixin' in dirC.lower(): continue if 'weibo' in dirC.lower(): continue if not cityShorten[dirC] in cities: continue print(' city: ', cityShorten[dirC], dirC) cityCount += 1 # City LN dfTTC = pd.DataFrame(columns=cs) dirCTs = os.listdir(os.path.join(path, dirC)) for dirCT in dirCTs: if dirCT[:1] == '.': continue # 时段 weibo weibo_1 if not os.path.isdir(os.path.join(path, dirC, dirCT)): continue if 'weixin' in dirCT.lower(): continue if 'weibo' in dirCT.lower(): continue if 'tt' in dirCT.lower(): print(' read TT... dir:',dirCT) fns = os.listdir(os.path.join(path, dirC, dirCT)) for fn in fns: if fn[:1] == '.': continue if not fn[-5:] == '.xlsx': continue #print('---',fn) # 账号名称 ttName = fn[fn.index('_')+1:] ttName = ttName[:ttName.index('_')] #D:\Projects\POM\DATA\2022年11月\10月报告\全文\LN\TT fileAs = os.path.join(path, dirC, dirCT, fn) #print(' ', ttName, fileAs) if len(fileAs) > 0: try: dfdftt = pd.read_excel(fileAs) dfTTC = dfTTC.append(dfdftt) except: print("read file failed. ", fileAs) #dfdfwb = pd.read_csv(filename, sep=',', header=None, names=cols, # index_col=None)#, engine='python', encoding='gbk'#utf-8 #dfdfwb = dfdfwb[1:] #dfdfwb["weiboID"] = wbId #dfdfwb["weiboName"] = wbName #dfTTC = dfTTC.append(dfdfwb) #print(ttName, '读入：', dfdftt.shape[0], ' 总计：', dfTTC.shape[0]) #if len(fileAs)>1: # print(" +=+= ", fileAs) print(' 读入头条数据行数', dfTTC.shape) #dfTTC.to_excel("D:/Projects/POM/2021年6月/二季度/全文/WB/WB_"+dirC+".xlsx") dfTTC['city'] = cityShorten[dirC] dfTT = dfTT.append(dfTTC) print('Read TT finished. cities', cityCount, '; lines', dfTT.shape) #dfTT.to_excel("D:/Projects/POM/2021年7月/2021年上半年/WB_ALL.xlsx") return dfTT def fetch_chinese(s): pattern =re.compile(r'[^\u4e00-\u9fa5]') sc = re.sub(pattern, '', s) return sc if __name__ == "__main__": starttime = datetime.datetime.now() _RATIO = 0.5 isDoWX = True isDoWB = True isDoTT = True cities = [ '临夏回族自治州', '白银市', '定西市', '酒泉市', '嘉峪关市', '平凉市', '庆阳市', '天水市', '武威市', '兰州新区', '陇南市', '兰州市', '张掖市', '甘南藏族自治州', '金昌市', '省直部门', # 共12市2州1新区 ] ''' cities = [ '临夏回族自治州', '白银市', '定西市', '酒泉市', '天水市', '陇南市', #'省直部门', # 共12市2州1新区 ] ''' #cities = ['陇南市', '临夏回族自治州', '白银市', '定西市', '酒泉市', '平凉市','武威市','天水市'] #cities = ['陇南市'] # 转发任务 #dfTask = pd.read_excel('D:/Projects/POM/DATA/2022年S2/S2/全省政务新媒体二季度转发信息条目.xls') dfTask = pd.read_excel('D:/Projects/POM/DATA/2023年4月/3月报告/2023年3月份全省政务新媒体转发内容条目.xlsx') # 账号信息 strFnAccount = 'D:/Projects/POM/DATA/2023年4月/3月报告/全国报送系统表单_2023.3.31.xlsx' dfAllAccount = pd.read_excel(strFnAccount) # 省直部门账号部门简称 dfProvincial = pd.read_excel('D:/Projects/POM/DATA/2023年4月/3月报告/省直部门账号名称简称.xlsx') fnTemplate = 'D:/Projects/POM/DATA/2023年4月/3月报告/POM_ForewardTemplate.docx' # 数据根目录， strPath = ['D:/Projects/POM/DATA/2023年4月/3月报告/'] strOutputPath = 'D:/Projects/POM/DATA/2023年4月/3月报告/转发/' context = { "year": "2023", "month": "3", "pubMonth": "4", "dateStart": "2023年3月1日", "dateEnd": "2023年3月31日" } dfAllAccount.loc[:, '转发数'] = 0 dfAllAccount.loc[:, '阅读数'] = 0 ################################################ # 创建存储矩阵 # 按照转发任务创建统计矩阵 colRR = ['市州', '类型', '账号名称', '单位名称', '省直部门', '区县', '转发数', '阅读数'] for ididid in dfTask['序号'][0:dfTask['内容'].count()].tolist(): #for ididid in range(1, dfTask['内容'].count()): colRR.append(str(ididid)) dfRR = pd.DataFrame(columns=colRR) # 每列一个转发任务，每行一个账号 # 用于保存每一条转发任务的账号和文章 dfO = pd.DataFrame(columns=['任务序号', '任务名称', '类型', '公众号', '日期', '内容', '链接', '市州'] ) ################################################ # countWxForewards = 0 countWbForewards = 0 countTtForewards = 0 # TT if isDoTT: print('=============================================================') print('---- TT ----') # id userId source city tid cellType title # time-stamp date url commentCount readNum likeNum showNum #dfTT = getTTData(strFnTT, strFnAccount, cities) #附加市州信息， cities未使用 dfTT = pd.DataFrame() for strP in strPath: ddff = getTTData(strP + '全文/', cities) print(" read TT data", ddff.shape) dfTT = dfTT.append(ddff) print("tt data ready", dfTT.shape) # cs = ['account', 'date', 'title', 'nread', 'ncomment', 'content', 'url', 'origin', 'city'] #cities = dfTT['city'].unique() for city in cities: dataC = dfTT.loc[dfTT['city'] == city].copy() accounts = dataC['account'].unique() dfdfCityTT = dfAllAccount.loc[(dfAllAccount['账号类型'] == '今日头条') & (dfAllAccount['市/省局'] == city)].copy() print(' count TT, city:', city, '读入账号数：', len(accounts), '任务账号数：', dfdfCityTT.shape[0]) print(' ', dataC.shape) for account in accounts: #一个公众号的所有文章 #print(account) dataA = dataC[dataC['account']==account] sR = pd.Series([], dtype=pd.StringDtype()) sR['类型'] = '今日头条' sR['市州'] = city sR['账号名称'] = account count = 0 for i in range(dfTask['内容'].count()): rn = dfTask.iloc[i, dfTask.columns.get_loc('序号')] rt = str(dfTask.iloc[i, dfTask.columns.get_loc('内容')]) #任务标题 forwarded = 0 for j in range(dataA.shape[0]): str1 = str(dataA.iloc[j, dataA.columns.get_loc('title')]) # 文章标题 # if len(rt) > len(str1): # 任务标题过长，截取前半部分进行对比 strRT = rt[:len(str1)] else: #文章标题过长，只比较任务标题长度部分 strRT = rt str1 = str1[:len(rt)] ratio = difflib.SequenceMatcher(None, strRT, str1).quick_ratio() if ratio > _RATIO: forwarded += 1 if forwarded > 0: break sR[str(rn)] = forwarded count += forwarded if forwarded > 0: dfO = dfO.append([{'任务序号': rn, '任务名称': rt, '类型': '今日头条', '公众号': account, #dfTT.iloc[j, dfTT.columns.get_loc('account')], '日期': dataA.iloc[j, dataA.columns.get_loc('date')], '内容': str1, '链接': dataA.iloc[j, dataA.columns.get_loc('url')], '市州': city, }], ignore_index=True) #checknames = ['白银高新区管委会', '健康白银', '白银市卫生健康委', '白银市卫生健康委员会', '白银发改委', '白银市场监管', '白银农业农村', '白银应急', '白银退役军人', '白银政务服务', '白银政务服务', '白银文旅', '白银科技', '白银林草', '白银市平川区商务局', '平川区长征街道', '平川区红会路街道', '和谐复兴', '兴平路街道办', '平川金融办'] #if account in checknames: # print(' -- ', account, count) sR['转发数'] = count dfRR = dfRR.append(sR, ignore_index=True, sort=False) #dfRR.to_excel('D:/Projects/POM/2021年6月/二季度/转发统计__'+city+'.xlsx') countTtForewards = dfRR.shape[0] - countWxForewards - countWbForewards print('TT forewards', countTtForewards) # WX if isDoWX: print('=============================================================') print('---- WX ----') dfWX = pd.DataFrame() for strP in strPath: ddff = getWXData(strP + '全文/', cities) print(' read WX data', ddff.shape) dfWX = dfWX.append(ddff) print('WX data ready', dfWX.shape) dfWX = dfWX.fillna(value=0) # 公众号 链接 日期 标题 内容 头条 city ## 逐个市州统计每个账号的转发情况 #cities = dfWX['市州'].unique() for city in cities: print(" count WX, city:", city) dataC = dfWX.loc[dfWX['市州'] == city].copy() accounts = dataC['公众号'].unique() dfdfCityWX = dfAllAccount.loc[((dfAllAccount['账号类型'] == '小程序+微信') | (dfAllAccount['账号类型'] == '微信服务号') | (dfAllAccount['账号类型'] == '微信订阅号')) & (dfAllAccount['市/省局'] == city)].copy() print(" count WX, city:", city, '账号数:', len(accounts), '任务账号数:', dfdfCityWX.shape[0]) for account in accounts: # print(account) dataA = dataC.loc[dataC['公众号'] == account].copy() # 一个公众号的所有文章 sR = pd.Series(dtype='object') sR['类型'] = '微信' sR['市州'] = city sR['账号名称'] = account count = 0 arn = 0 for i in range(dfTask['内容'].count()): # 对于每一篇任务文章 rn = dfTask.iloc[i, dfTask.columns.get_loc('序号')] rt = str(dfTask.iloc[i, dfTask.columns.get_loc('内容')]) forwarded = 0 readNum = 0 # 查看该账号的所有文章 for j in range(dataA.shape[0]): str1 = str(dataA.iloc[j, dataA.columns.get_loc('标题')]) # if len(rt) > len(str1):# 任务标题过长，截取前半部分进行对比 strRT = rt[:len(str1)] else:#文章标题过长，只比较任务标题长度部分 strRT = rt str1 = str1[:len(rt)] ratio = difflib.SequenceMatcher(None, strRT, str1).quick_ratio() tRN = int(dataA.iloc[j, dataA.columns.get_loc('阅读数')]) # 遇到相似的即跳出 if ratio > _RATIO: forwarded += 1 readNum += tRN if forwarded > 0: break sR[str(rn)] = forwarded count += forwarded arn += readNum if forwarded > 0: dfO = dfO.append([{'任务序号': rn, '任务名称': rt, '类型': '微信', '公众号': account, '日期': dataA.iloc[j, dataA.columns.get_loc('日期')], '内容': str1, '链接': dataA.iloc[j, dataA.columns.get_loc('链接')], '市州': city, '阅读数': readNum, }], ignore_index=True) sR['转发数'] = count sR['阅读数'] = arn dfRR = dfRR.append(sR, ignore_index=True, sort=False) countWxForewards = dfRR.shape[0] print('WX forwards', countWxForewards) # WB if isDoWB: print('=============================================================') print('---- WB ----') dfWB = pd.DataFrame() for strP in strPath: ddff = getWBData(strP + '全文/', cities) print(' read WB data', ddff.shape) dfWB = dfWB.append(ddff) print('WB data ready', dfWB.shape) ################################################ # WB # 微博id 微博正文 头条文章url 原始图片url 被转发微博原始图片url 是否为原创微博 微博视频url 发布位置 date # 发布工具 点赞数 转发数 评论数 weiboID weiboName city #cities = dfWB['市州'].unique() for city in cities: print(' count WB, city:', city) dataC = dfWB.loc[dfWB['市州'] == city].copy() accounts = dataC['weiboName'].unique() dfdfCityWB = dfAllAccount.loc[(dfAllAccount['账号类型'] == '新浪微博') & (dfAllAccount['市/省局'] == city)].copy() print(' count WB, city:', city, "读入账号数：", len(accounts), '任务账号数：', dfdfCityWB.shape[0]) for account in accounts: # 一个公众号的所有文章 # print(account) dataA = dataC.loc[dataC['weiboName'] == account].copy() sR = pd.Series(dtype='object') sR['类型'] = '新浪微博' sR['市州'] = city sR['账号名称'] = account count = 0 # 对一个账号，用任务标题从它全部发文里比对 # 若找到， 该任务标记为已转发 for i in range(dfTask['内容'].count()): rn = dfTask.iloc[i, dfTask.columns.get_loc('序号')] rt = str(dfTask.iloc[i, dfTask.columns.get_loc('内容')]) forwarded = 0 for j in range(dataA.shape[0]): str1 = str(dataA.iloc[j, dataA.columns.get_loc('微博正文')]) str2 = str1[:len(rt)] #if rt in str1: ratio = difflib.SequenceMatcher(None, fetch_chinese(rt), fetch_chinese(str2)).quick_ratio() # if account=='陇南公积金' and ratio > 0.5: # print('----', ratio) # print(rt) # print(fetch_chinese(rt)) # print(str2) # print(fetch_chinese(str2)) if ratio > _RATIO: forwarded += 1 if forwarded > 0: break sR[str(rn)] = forwarded count += forwarded if forwarded > 0: dfO = dfO.append([{'任务序号': rn, '任务名称': rt, '类型': '新浪微博', '公众号': account, '日期': dataA.iloc[j, dataA.columns.get_loc('date')], '内容': str1, '链接': dataA.iloc[j, dataA.columns.get_loc('头条文章url')], '市州': city, }], ignore_index=True) sR['转发数'] = count ##print(account, count) dfRR = dfRR.append(sR, ignore_index=True, sort=False) ####if account=='陇南公积金' : ####print(account, dataA.shape[0], count) # dfRR.to_excel('D:/Projects/POM/2021年6月/二季度/转发统计__'+city+'.xlsx') countWbForewards = dfRR.shape[0] - countWxForewards print('WB forwards', countWbForewards) #dfRR.to_excel(strOutputPath + '转发统计_GS_ALL.xlsx') #dfO.to_excel(strOutputPath + '转发统计_GS_ALLDATA.xlsx') if isDoWX or isDoWB or isDoTT: print('=============================================================') print('---- STATISTICS ----') print('=============================================================') print('ALL forewards account num:', dfRR.shape[0], 'task num:', dfRR.shape[1]) ################################################ # 统计每一个账号的累计转发数和阅读数 # ~~ 匹配到账号所有信息的表格中 # dfAAWX = dfAllAccount.loc[(dfAllAccount['账号类型'] == '小程序+微信') | (dfAllAccount['账号类型'] == '微信服务号') | (dfAllAccount['账号类型'] == '微信订阅号') ].copy() dfRRWX = dfRR.loc[dfRR['类型'] == '微信'].copy() dfRRWX.rename(columns={'账号名称':'account'}, inplace=True) ## 遍历账号详情表 for i in range(dfAAWX.shape[0]): strName = str(dfAAWX.iloc[i, dfAAWX.columns.get_loc('账号名称')]) dfRRRRWX = dfRRWX[dfRRWX.account == strName] aName = re.sub('\s+','', str(dfAAWX.iloc[i, dfAAWX.columns.get_loc('账号名称')])) matchedRow = -1 # 转发统计表中的行数 # 对每一个微信账号，从转发统计表中匹配账号名称 for j in range(dfRRWX.shape[0]): name = re.sub('\s+', '', str(dfRRWX.iloc[j, dfRRWX.columns.get_loc('account')])) if name == aName: matchedRow = j break if matchedRow > -1 : r = int(dfRRWX.iloc[matchedRow, dfRRWX.columns.get_loc('转发数')]) readNum = int(dfRRWX.iloc[matchedRow, dfRRWX.columns.get_loc('阅读数')]) dfAAWX.iloc[i, dfAAWX.columns.get_loc('转发数')] = r dfAAWX.iloc[i, dfAAWX.columns.get_loc('阅读数')] = readNum if dfRRRRWX.shape[0]>0 and matchedRow<0: print(' -', aName, strName, dfRRRRWX.shape[0], matchedRow) ###### if dfRRRRWX.shape[0]<1 and matchedRow>-1: print(' =', aName, strName, dfRRRRWX.shape[0], matchedRow) ###### #dfAAWX.to_excel('D:/Projects/POM/2021年7月/月报告/6月份全文数据/转发统计___List.xlsx') print('总微信账号数：', dfAAWX.shape) dfAAWB = dfAllAccount.loc[dfAllAccount['账号类型'] == '新浪微博'].copy() dfRRWB = dfRR.loc[dfRR['类型'] == '新浪微博'].copy() for i in range(dfAAWB.shape[0]): aName = re.sub('\s+','', str(dfAAWB.iloc[i, dfAAWB.columns.get_loc('账号名称')])) matchedRow = -1 # 对每一个账号，从转发统计表中匹配账号名称 for j in range(dfRRWB.shape[0]): name = re.sub('\s+', '', str(dfRRWB.iloc[j, dfRRWB.columns.get_loc('账号名称')])) if name == aName: matchedRow = j break if matchedRow > -1 : r = int(dfRRWB.iloc[matchedRow, dfRRWB.columns.get_loc('转发数')]) dfAAWB.iloc[i, dfAAWB.columns.get_loc('转发数')] = r print('总微博账号数：', dfAAWB.shape) dfAATT = dfAllAccount.loc[dfAllAccount['账号类型'] == '今日头条'].copy() dfRRTT = dfRR.loc[dfRR['类型'] == '今日头条'].copy() #'市州', '类型', '账号名称', '转发数' dfRRTT.rename(columns={'账号名称':'account'}, inplace=True) for i in range(dfAATT.shape[0]): aName = re.sub('\s+','', str(dfAATT.iloc[i, dfAATT.columns.get_loc('账号名称')])) strName = str(dfAATT.iloc[i, dfAATT.columns.get_loc('账号名称')]) dfRRRRTT = dfRRTT[dfRRTT.account == strName] matchedRow = -1 #dftmp = dfRRTT[dfRRTT['账号名称']==aName] for j in range(dfRRTT.shape[0]): name = re.sub('\s+', '', str(dfRRTT.iloc[j, dfRRTT.columns.get_loc('account')])) if name == aName: matchedRow = j break if matchedRow > -1 : r = int(dfRRTT.iloc[matchedRow, dfRRTT.columns.get_loc('转发数')]) ###### ###### #dfAATT.iloc[i, dfAATT.columns.get_loc('转发数')] = r dfAATT.loc[i, dfAATT.columns.get_loc('转发数')] = r if dfRRRRTT.shape[0]>0 and matchedRow<0: print(' -', aName, strName, dfRRRRTT.shape[0], matchedRow) ###### if dfRRRRTT.shape[0]<1 and matchedRow>-1: print(' =', aName, strName, dfRRRRTT.shape[0], matchedRow) ###### print('总头条账号数：', dfAATT.shape) dfAAA = dfAAWX.append(dfAAWB, ignore_index=True, sort=False) dfAAA = dfAAA.append(dfAATT, ignore_index=True, sort=False) print('所有统计账号数：', dfAAA.shape) ################################################ # 为转发账号匹配单位全称和所属县区 # for i in range(dfRR.shape[0]): aName = re.sub('\s+','', str(dfRR.iloc[i, dfRR.columns.get_loc('账号名称')])) matchedRow = -1 # 区县/地方部门 # 对每一个账号，从表中匹配账号名称 for j in range(dfAllAccount.shape[0]): name = re.sub('\s+', '', str(dfAllAccount.iloc[j, dfAllAccount.columns.get_loc('账号名称')])) if name == aName: matchedRow = j break if matchedRow > -1 : cc = str(dfAllAccount.iloc[matchedRow, dfAllAccount.columns.get_loc('区县/地方部门')]) if cc != 'nan': dfRR.iloc[i, dfRR.columns.get_loc('区县')] = cc bn = str(dfAllAccount.iloc[matchedRow, dfAllAccount.columns.get_loc('单位全称')]) if bn != 'nan': dfRR.iloc[i, dfRR.columns.get_loc('单位名称')] = bn ###### # 为省级部门匹配简称 if '省直部门' in cities: city = '省直部门' dfAAACity = dfAAA[dfAAA['市/省局'] == city] dfAAACity.to_excel(strOutputPath + '账号转发量_' + city + '.xlsx') dfOCity = dfO[dfO['市州'] == city] dfO.to_excel(strOutputPath + '转发文章_' + city + '.xlsx') dfRRCity = dfRR[dfRR['市州'] == city] if city in ['临夏回族自治州', '甘南藏族自治州']: dfRRCity['区县'].fillna('州直部门', inplace=True) else: dfRRCity['区县'].fillna('市直部门', inplace=True) dfRRCity.to_excel(strOutputPath + '转发账号_' + city + '.xlsx') dfRR.to_excel(strOutputPath + '转发账号.xlsx') dfAAA.to_excel(strOutputPath + '账号转发量.xlsx') dfO.to_excel(strOutputPath + '转发文章.xlsx') # dfAAA = pd.read_excel(strOutputPath + '账号转发量.xlsx') # dfRR = pd.read_excel(strOutputPath + '转发账号.xlsx') # dfO = pd.read_excel(strOutputPath + '转发文章.xlsx') # 过长名称替换为简称，便于绘图 dfRR.loc[dfRR['区县'] == '积石山保安族东乡族撒拉族自治县', '区县'] = '积石山县' dfRR.loc[dfRR['区县'] == '阿克塞哈萨克族自治县', '区县'] = '阿克塞自治县' for city in cities: if city in ['兰州新区', '省直部门']: continue print(" add up city", city) ###### # 匹配省级部门的简称 if city == '省直部门': for i in range(dfRR.shape[0]): aName = re.sub('\s+', '', str(dfRR.iloc[i, dfRR.columns.get_loc('账号名称')])) matchedRow = -1 # 对每一个账号，从表中匹配账号名称 for j in range(dfProvincial.shape[0]): name = re.sub('\s+', '', str(dfProvincial.iloc[j, dfProvincial.columns.get_loc('账号名称')])) if name == aName: matchedRow = j break if matchedRow > -1: bmjc = str(dfProvincial.iloc[matchedRow, dfProvincial.columns.get_loc('简称')]) if bmjc != 'nan': dfRR.iloc[i, dfRR.columns.get_loc('省直部门')] = bmjc dfAAACity = dfAAA.loc[dfAAA['市/省局'] == city].copy() dfAAACity.to_excel(strOutputPath + '账号转发量_' + city + '.xlsx') dfOCity = dfO[dfO['市州'] == city] dfO.to_excel(strOutputPath + '转发文章_' + city + '.xlsx') dfRRCity = dfRR.loc[dfRR['市州'] == city].copy() if city in ['临夏回族自治州', '甘南藏族自治州']: dfRRCity['区县'].fillna('州直部门', inplace=True) dfRRCityD = dfRRCity.loc[dfRRCity['区县'] == '州直部门'].copy() else: dfRRCity['区县'].fillna('市直部门', inplace=True) dfRRCityD = dfRRCity.loc[dfRRCity['区县'] == '市直部门'].copy() dfRRCity.to_excel(strOutputPath + '转发账号_' + city + '.xlsx') #### # 统计市直部门 #dfRRCityD = dfRRCity[dfRRCity['区县'] == '州直部门'] dfRRCD1 = pd.pivot_table(dfRRCityD, index=['单位名称'], values=['账号名称'], aggfunc = ['count'], fill_value='', margins=True, margins_name='总计') dfRRCD2 = pd.pivot_table(dfRRCityD, index=['单位名称'], values=['转发数'], aggfunc = ['sum'], fill_value='', margins=True, margins_name='总计') dfRRCD3 = pd.pivot_table(dfRRCityD, index=['单位名称'], values=['阅读数'], aggfunc = ['sum'], fill_value='', margins=True, margins_name='总计') dfRRCD = pd.concat([dfRRCD1, dfRRCD2, dfRRCD3], axis=1) # 计算转发率 dfRRCD.columns = ['_'.join(col) for col in dfRRCD.columns.values] #dfRRCD.reset_index(inplace=True) dfRRCD['rate'] = dfRRCD.apply(lambda x: int(x['sum_转发数'] / x['count_账号名称'] / dfTask['内容'].count() * 1000)/1000.0, axis=1) # 排序 dfRRCD = dfRRCD[0:dfRRCD.shape[0]-1].sort_values(by='rate', ascending=False) dfRRCD = pd.concat([dfRRCD, dfRRCD[dfRRCD.shape[0]-1:dfRRCD.shape[0]] ], axis=0) dfRRCD.to_excel(strOutputPath + '市州直部门转发_' + city + '.xlsx') ####### # 统计县区 # 发现目前版本pivot_table函数aggfunc用列表时，前几列计算值不准确 # 所以，暂时单列计算，再合并 dfCountyA = pd.pivot_table(dfRRCity, index=['区县'], values=['账号名称'], aggfunc = ['count'], fill_value='', margins=True, margins_name='总计') dfCountyC = pd.pivot_table(dfRRCity, index=['区县'], values=['转发数'], aggfunc = ['sum'], fill_value='', margins=True, margins_name='总计') dfCountyR = pd.pivot_table(dfRRCity, index=['区县'], values=['阅读数'], aggfunc = ['sum'], fill_value='', margins=True, margins_name='总计') dfCounty = pd.concat([dfCountyA, dfCountyC, dfCountyR], axis=1) # 计算转发率 dfCounty.columns = ['_'.join(col) for col in dfCounty.columns.values] #dfCounty.reset_index(inplace=True) dfCounty['rate'] = dfCounty.apply(lambda x: int(x['sum_转发数'] / x['count_账号名称'] / dfTask['内容'].count() * 1000)/1000.0, axis=1) # 排序 dfCC = dfCounty[0:dfCounty.shape[0]-1].sort_values(by='rate', ascending=False) dfCC = pd.concat([dfCC, dfCounty[dfCounty.shape[0]-1:dfCounty.shape[0]] ], axis=0) dfCC.to_excel(strOutputPath + '县区转发_' + city + '.xlsx') # 统计市/州直部门转发数 dfRRD = dfRRCity[(dfRRCity['区县'] == '州直部门') | (dfRRCity['区县'] == '市直部门')] dfDA = pd.pivot_table(dfRRD, index=['单位名称'], values=['账号名称'], aggfunc=['count'], fill_value='', margins=True, margins_name='总计') dfDC = pd.pivot_table(dfRRD, index=['单位名称'], values=['转发数'], aggfunc=['sum'], fill_value='', margins=True, margins_name='总计') dfDR = pd.pivot_table(dfRRD, index=['单位名称'], values=['阅读数'], aggfunc=['sum'], fill_value='', margins=True, margins_name='总计') dfD = pd.concat([dfDA, dfDC, dfDR], axis=1) # 计算部门转发率 #print(dfD.columns) # 合并多层索引MultiIndex dfD.columns = ['_'.join(col) for col in dfD.columns.values] #dfD.reset_index(inplace=True) # 计算转发率 dfD['rate'] = dfD.apply( lambda x: int(x['sum_转发数'] / x['count_账号名称'] / dfTask['内容'].count() * 1000) / 1000.0, axis=1) # 排序 dfDD = dfD[0:dfD.shape[0] - 1].sort_values(by='rate', ascending=False) dfDD = pd.concat([dfDD, dfD[dfD.shape[0] - 1:dfD.shape[0]]], axis=0) dfDD.to_excel(strOutputPath + '部门转发_' + city + '.xlsx') ######################################################### # # 生成报告 tpl = DocxTemplate(fnTemplate) info = { "taskCount": dfTask['内容'].count(), "aNum": int(dfCC.iloc[-1]['count_账号名称']), "fNum": int(dfCC.iloc[-1]['sum_转发数']), "readNum": int(dfCC.iloc[-1]['sum_阅读数']), "r": '%.1f'%(dfCC.iloc[-1]['rate']*100.0), # "dNum": int(dfDD.iloc[-1]['count_账号名称']), # 部门总账号数 "dFNum": int(dfDD.iloc[-1]['sum_转发数']), # 部门总转发数 "dReadNum": int(dfDD.iloc[-1]['sum_阅读数']), # 部门总阅读数 "dr": '%.1f'%(dfDD.iloc[-1]['rate']*100.0), # 部门平均转发率 } context.update(info) # 县区转发率表格 t1_list = [] for index, row in dfCC.iterrows(): if index == "总计": continue t1_a = {'county': str(index), 'rate': '%.1f'%(row['rate']*100.0), 'account': int(row['count_账号名称']), 'fNum': int(row['sum_转发数']), 'readNum': int(row['sum_阅读数'])} t1_list.append(t1_a) context['t1_contents'] = t1_list # 部门转发率表格 t2_list = [] for index, row in dfDD.iterrows(): if index == "总计": continue t2_a = {'name': str(index), 'rate': '%.1f'%(row['rate']*100.0), 'account': int(row['count_账号名称']), 'fNum': int(row['sum_转发数']), 'readNum': int(row['sum_阅读数'])} t2_list.append(t2_a) context['t2_contents'] = t2_list # 转发任务列表 t3_list = [] for index, row in dfTask.iterrows(): t3_a = {'id': row['序号'], 'title': row['内容'], 'date': row['时间'].strftime('%Y-%m-%d') } t3_list.append(t3_a) context['t3_contents'] = t3_list # 绘制区县转发率图 drawBar(dfCC['rate'][:-1], dfCC.index[:-1], '区县转发率', os.path.join(strOutputPath, city + '_graphCounty.png')) dc = { 'graphCounty': InlineImage(tpl, os.path.join(strOutputPath, city+'_graphCounty.png'), width=Mm(120)), } context.update(dc) tpl.render(context) tpl.save(strOutputPath+city+'.docx') ###### #### ####### ###### #### ####### ###### #### ####### # 统计县区 # 发现目前版本pivot_table函数aggfunc用列表时，前几列计算值不准确 # 所以，暂时单列计算，再合并 dfCountyA = pd.pivot_table(dfRR, index=['市州'], values=['账号名称'], aggfunc = ['count'], fill_value='', margins=True, margins_name='总计') dfCountyC = pd.pivot_table(dfRR, index=['市州'], values=['转发数'], aggfunc = ['sum'], fill_value='', margins=True, margins_name='总计') dfCountyR = pd.pivot_table(dfRR, index=['市州'], values=['阅读数'], aggfunc = ['sum'], fill_value='', margins=True, margins_name='总计') dfCounty = pd.concat([dfCountyA, dfCountyC, dfCountyR], axis=1) # 计算转发率 dfCounty.columns = ['_'.join(col) for col in dfCounty.columns.values] #dfCounty.reset_index(inplace=True) dfCounty['rate'] = dfCounty.apply(lambda x: int(x['sum_转发数'] / x['count_账号名称'] / dfTask['内容'].count() * 1000)/1000.0, axis=1) # 排序 dfCC = dfCounty[0:dfCounty.shape[0]-1].sort_values(by='rate', ascending=False) dfCC = pd.concat([dfCC, dfCounty[dfCounty.shape[0]-1:dfCounty.shape[0]] ], axis=0) dfCC.to_excel(strOutputPath + '市州转发_ALL.xlsx') # 统计市/州直部门转发数 #if '省直部门' in cities: dfRRD = dfRR[ (dfRR['市州'] == '省直部门')] dfDD = pd.DataFrame() if dfRRD.shape[0] > 0: dfDA = pd.pivot_table(dfRRD, index=['单位名称'], values=['账号名称'], aggfunc=['count'], fill_value='', margins=True, margins_name='总计') dfDC = pd.pivot_table(dfRRD, index=['单位名称'], values=['转发数'], aggfunc=['sum'], fill_value='', margins=True, margins_name='总计') dfDR = pd.pivot_table(dfRRD, index=['单位名称'], values=['阅读数'], aggfunc=['sum'], fill_value='', margins=True, margins_name='总计') dfD = pd.concat([dfDA, dfDC, dfDR], axis=1) # 计算部门转发率 print('---', dfD.columns) # 合并多层索引MultiIndex dfD.columns = ['_'.join(col) for col in dfD.columns.values] #dfD.reset_index(inplace=True) print('---', dfD.columns) # 计算转发率 dfD['rate'] = dfD.apply( lambda x: int(x['sum_转发数'] / x['count_账号名称'] / dfTask['内容'].count() * 1000) / 1000.0, axis=1) # 排序 dfDD = dfD[0:dfD.shape[0] - 1].sort_values(by='rate', ascending=False) dfDD = pd.concat([dfDD, dfD[dfD.shape[0] - 1:dfD.shape[0]]], axis=0) dfDD.to_excel(strOutputPath + '部门转发_ALL.xlsx') ######################################################### # # 生成报告 tpl = DocxTemplate(fnTemplate) info = { "taskCount": dfTask['内容'].count(), "aNum": int(dfCC.iloc[-1]['count_账号名称']), "fNum": int(dfCC.iloc[-1]['sum_转发数']), "readNum": int(dfCC.iloc[-1]['sum_阅读数']), "r": '%.1f'%(dfCC.iloc[-1]['rate']*100.0), } if dfDD.empty: info.update( { "dNum": 0, # 部门总账号数 "dFNum": 0, # 部门总转发数 "dReadNum": 0, # 部门总阅读数 "dr": '%.1f'%(0), # 部门平均转发率 } ) else: info.update( { "dNum": int(dfDD.iloc[-1]['count_账号名称']), # 部门总账号数 "dFNum": int(dfDD.iloc[-1]['sum_转发数']), # 部门总转发数 "dReadNum": int(dfDD.iloc[-1]['sum_阅读数']), # 部门总阅读数 "dr": '%.1f'%(dfDD.iloc[-1]['rate']*100.0), # 部门平均转发率 }) context.update(info) # 县区转发率表格 t1_list = [] for index, row in dfCC.iterrows(): if index == "总计": continue t1_a = {'county': str(index), #str('账号名称'), 'rate': '%.1f'%(row['rate']*100.0), 'account': int(row['count_账号名称']), 'fNum': int(row['sum_转发数']), 'readNum': int(row['sum_阅读数'])} t1_list.append(t1_a) context['t1_contents'] = t1_list # 部门转发率表格 t2_list = [] if not dfDD.empty: for index, row in dfDD.iterrows(): if index == "总计": continue t2_a = {'name': str(index), #str(row['单位名称']), 'rate': '%.1f'%(row['rate']*100.0), 'account': int(row['count_账号名称']), 'fNum': int(row['sum_转发数']), 'readNum': int(row['sum_阅读数'])} t2_list.append(t2_a) context['t2_contents'] = t2_list # 转发任务列表 t3_list = [] for index, row in dfTask.iterrows(): t3_a = {'id': row['序号'], 'title': row['内容'], 'date': row['时间'].strftime('%Y-%m-%d') } t3_list.append(t3_a) context['t3_contents'] = t3_list # 绘制区县转发率图 drawBar(dfCC['rate'][:-1], dfCC.index[:-1], '市州转发率', os.path.join(strOutputPath, 'ALL_graphCounty.png')) dc = { 'graphCounty': InlineImage(tpl, os.path.join(strOutputPath, 'ALL_graphCounty.png'), width=Mm(120)), } context.update(dc) tpl.render(context) tpl.save(strOutputPath+'ALL.docx') endtime = datetime.datetime.now() usedtime = endtime - starttime print("time: ", usedtime)