bob
/
newmediamonitoring
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

#数据更新

This commit is contained in:
xieshen 2021-01-07 18:51:58 +08:00
parent 4cbdd65caa
commit be6f49ea0f
13 changed files with 687 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
新媒体更新脚本/test.py Normal file
View File

@ -0,0 +1,7 @@
import re
s = 'kkkk kojoi' \
'koko'
r = re.sub('\s+', '', s).strip()
print(r)

64
新媒体更新脚本/更新其他名称.py Normal file
View File

@ -0,0 +1,64 @@
# 更新新媒体脚本，结果为有出入的新媒体列表。
'''
1,将excel转为csv，使用utf-8编码集。
2、获取csv内容
3、与数据库中的数据逐项对比
'''
import csv
import re
import uuid
import psycopg2 as psycopg2
# code = None
# alias = None
# attention = None
# remark = None
# identificationcode = None
# function = None
# articleurl = None
# weixinid = None
# type = None
G2 = 'host=210.77.68.250 port=5432 dbname=newmediaDB3 user=newmedia password=newmedia2020!@#'
def set_csv_weixin(path):
with open(path, encoding='utf-8') as csvfile:
reader = csv.reader(csvfile)
for r in reader:
if r[1] == '删除':
print(r)
# with psycopg2.connect(G2) as connection:
# with connection.cursor() as cursor:
# cursor.execute(
# "delete from dashboard_qita where code = '%s'" %(re.sub('\s+', '', r[0]).strip()))
# connection.commit()
elif r[1] == '新增':
with open('D:/2020/新媒体监测/TASK_ALL.csv',encoding='utf-8') as c:
t = csv.reader(c)
for i in t:
print(i[4],r[0])
if re.sub('\s+', '', r[0]).strip() == re.sub('\s+', '', i[4]).strip() and re.sub('\s+', '', r[2]).strip() == re.sub('\s+', '', i[7]).strip():
with psycopg2.connect(G2) as connection:
with connection.cursor() as cursor:
try:
cursor.execute(
"select * from dashboard_organization where name = '%s'" % (
re.sub('\s+', '', i[1]).strip()))
connection.commit()
print(i[1] + '++++++++++++++++++++++++++++++++++++++++')
for c in cursor:
print(c[0] + "22222222222222222222222222222222222")
id = uuid.uuid4()
cursor.execute(
"insert into dashboard_qita(id,code, created, updated,organization_id,image,alias,status,qitaid,attention, remark,identificationcode,function,articleurl,type) values (%s,%s,now(),now(),%s,'qita.png','',3,%s,'','',%s,%s,%s,%s)",
(str(id), re.sub('\s+', '', i[4]).strip(), c[0], i[13], i[3], i[8], i[12],i[7]))
connection.commit()
except Exception as e:
print(e)
if __name__ == '__main__':
set_csv_weixin('D:/2020/新媒体监测/新媒体对比结果_其他_账号名称更改.csv')

54
新媒体更新脚本/更新单位名称.py Normal file
View File

@ -0,0 +1,54 @@
# 更新新媒体脚本，结果为有出入的新媒体列表。
'''
1,将excel转为csv，使用utf-8编码集。
2、获取csv内容
3、与数据库中的数据逐项对比
'''
import csv
import uuid
import psycopg2 as psycopg2
# code = None
# alias = None
# attention = None
# remark = None
# identificationcode = None
# function = None
# articleurl = None
# weixinid = None
# type = None
G2 = 'host=210.77.68.250 port=5432 dbname=newmediaDB3 user=newmedia password=newmedia2020!@#'
def set_csv_weixin(path):
with open(path, encoding='utf-8') as csvfile:
reader = csv.reader(csvfile)
for r in reader:
if r[1] == '删除':
print(r)
# with psycopg2.connect(G2) as connection:
# with connection.cursor() as cursor:
# cursor.execute(
# "delete from dashboard_organization where name = '%s'" %(r[0]))
# connection.commit()
elif r[1] == '新增':
with open('D:/2020/新媒体监测/TASK_ALL.csv',encoding='utf-8') as c:
t = csv.reader(c)
for i in t:
if r[0].replace(' ', '') == i[1].replace(' ', ''):
print(i[1], r[0])
with psycopg2.connect(G2) as connection:
with connection.cursor() as cursor:
id = uuid.uuid4()
cursor.execute(
"insert into dashboard_organization(id,name,image, province, cities, district, town,village,directly,status,level_id,id_code, created, updated) values (%s,%s, 'danweimoren.jpg','620000000000',%s,%s,'','', %s,'1','69be9ef4-b7b7-4049-a86e-7083bee40f0e',%s,now(),now())"
, (str(id), i[1],i[10],i[11],i[6],i[2]))
connection.commit()
if __name__ == '__main__':
set_csv_weixin('D:/2020/新媒体监测/单位名称对比结果.csv')

64
新媒体更新脚本/更新头条名称.py Normal file
View File

@ -0,0 +1,64 @@
# 更新新媒体脚本，结果为有出入的新媒体列表。
'''
1,将excel转为csv，使用utf-8编码集。
2、获取csv内容
3、与数据库中的数据逐项对比
'''
import csv
import re
import uuid
import psycopg2 as psycopg2
# code = None
# alias = None
# attention = None
# remark = None
# identificationcode = None
# function = None
# articleurl = None
# weixinid = None
# type = None
G2 = 'host=210.77.68.250 port=5432 dbname=newmediaDB3 user=newmedia password=newmedia2020!@#'
def set_csv_weixin(path):
with open(path, encoding='utf-8') as csvfile:
reader = csv.reader(csvfile)
for r in reader:
if r[1] == '删除':
print(r)
with psycopg2.connect(G2) as connection:
with connection.cursor() as cursor:
cursor.execute(
"delete from dashboard_toutiao where code = '%s'" %(re.sub('\s+', '', r[0]).strip()))
connection.commit()
elif r[1] == '新增':
with open('D:/2020/新媒体监测/TASK_ALL.csv',encoding='utf-8') as c:
t = csv.reader(c)
for i in t:
print(i[4],r[0])
if re.sub('\s+', '', r[0]).strip() == re.sub('\s+', '', i[4]).strip():
with psycopg2.connect(G2) as connection:
with connection.cursor() as cursor:
try:
cursor.execute(
"select * from dashboard_organization where name = '%s'" % (
re.sub('\s+', '', i[1]).strip()))
connection.commit()
print(i[1] + '++++++++++++++++++++++++++++++++++++++++')
for c in cursor:
print(c[0] + "22222222222print(i[4],r[0])222222222222222222222222")
id = uuid.uuid4()
cursor.execute(
"insert into dashboard_toutiao(id,code, created, updated,organization_id,image,alias,status,toutiaoid,attention, remark,identificationcode,function,articleurl) values (%s,%s,now(),now(),%s,'toutiao.png','',3,%s,'','',%s,%s,%s)",
(str(id), re.sub('\s+', '', i[4]).strip(), c[0], i[13], i[3], i[8], i[12]))
connection.commit()
except Exception as e:
print(e)
if __name__ == '__main__':
set_csv_weixin('D:/2020/新媒体监测/新媒体对比结果_头条_账号名称更改.csv')

69
新媒体更新脚本/更新微信名称.py Normal file
View File

@ -0,0 +1,69 @@
# 更新新媒体脚本，结果为有出入的新媒体列表。
'''
1,将excel转为csv，使用utf-8编码集。
2、获取csv内容
3、与数据库中的数据逐项对比
'''
import csv
import re
import uuid
import psycopg2 as psycopg2
# code = None
# alias = None
# attention = None
# remark = None
# identificationcode = None
# function = None
# articleurl = None
# weixinid = None
# type = None
G2 = 'host=210.77.68.250 port=5432 dbname=newmediaDB3 user=newmedia password=newmedia2020!@#'
def set_csv_weixin(path):
with open(path, encoding='utf-8') as csvfile:
reader = csv.reader(csvfile)
for r in reader:
if r[1] == '删除':
with psycopg2.connect(G2) as connection:
with connection.cursor() as cursor:
cursor.execute(
"delete from dashboard_weixin where code = '%s'" %(re.sub('\s+', '', r[0]).strip()))
connection.commit()
elif r[1] == '新增':
# s = str(r[2]).split('[')[1].split(']')[0].replace("'","").split(',')
with open('D:/2020/新媒体监测/TASK_ALL.csv',encoding='utf-8') as c:
t = csv.reader(c)
for i in t:
# print(i[4],r[0])
if re.sub('\s+', '', r[0]).strip() == re.sub('\s+', '', i[4]).strip():
with psycopg2.connect(G2) as connection:
with connection.cursor() as cursor:
try:
cursor.execute(
"select * from dashboard_organization where name = '%s'" % (
re.sub('\s+', '', i[1]).strip()))
connection.commit()
for c in cursor:
# print(c[0] + "22222222222222222222222222222222222")
id = uuid.uuid4()
cursor.execute(
"insert into dashboard_weixin(id,code, created, updated,organization_id,image,alias,status,weixinid,attention, remark,identificationcode,function,articleurl) values (%s,%s,now(),now(),%s,'weixin.png','',3,%s,'','',%s,%s,%s)",
(str(id), re.sub('\s+', '', i[4]).strip(), c[0], i[13], i[3], i[8], i[12]))
connection.commit()
except Exception as e:
print(i[1]+"+++++++++++++++++++++++++++++++++++++++++++++++")
with open('D:/2020/新媒体监测/单位对不上.csv', "w", newline='',encoding='utf-8') as f:
writer = csv.writer(f)
writer.writerow(
i[1])
if __name__ == '__main__':
set_csv_weixin('D:/2020/新媒体监测/新媒体对比结果_微信_账号名称更改.csv')

62
新媒体更新脚本/更新微博名称.py Normal file
View File

@ -0,0 +1,62 @@
# 更新新媒体脚本，结果为有出入的新媒体列表。
'''
1,将excel转为csv，使用utf-8编码集。
2、获取csv内容
3、与数据库中的数据逐项对比
'''
import csv
import re
import uuid
import psycopg2 as psycopg2
# code = None
# alias = None
# attention = None
# remark = None
# identificationcode = None
# function = None
# articleurl = None
# weixinid = None
# type = None
G2 = 'host=210.77.68.250 port=5432 dbname=newmediaDB3 user=newmedia password=newmedia2020!@#'
def set_csv_weixin(path):
with open(path, encoding='utf-8') as csvfile:
reader = csv.reader(csvfile)
for r in reader:
if r[1] == '删除':
pass
# with psycopg2.connect(G2) as connection:
# with connection.cursor() as cursor:
# cursor.execute(
# "delete from dashboard_weibo where code = '%s'" %(re.sub('\s+', '', r[0]).strip()))
# connection.commit()
elif r[1] == '新增':
with open('D:/2020/新媒体监测/TASK_ALL.csv',encoding='utf-8') as c:
t = csv.reader(c)
for i in t:
if re.sub('\s+', '', r[0]).strip() == re.sub('\s+', '', i[4]).strip():
print(i[4], r[0])
with psycopg2.connect(G2) as connection:
with connection.cursor() as cursor:
try:
cursor.execute(
"select * from dashboard_organization where name = '%s'" % (
re.sub('\s+', '', i[1]).strip()))
connection.commit()
for c in cursor:
id = uuid.uuid4()
cursor.execute(
"insert into dashboard_weibo(id,code, created, updated,organization_id,image,alias,status,weiboid,attention, remark,identificationcode,function,articleurl) values (%s,%s,now(),now(),%s,'weibo.png','',3,%s,'','',%s,%s,%s)",
(str(id), re.sub('\s+', '', i[4]).strip(), c[0], i[13], i[3], i[8], i[12]))
connection.commit()
except Exception as e:
print(e)
if __name__ == '__main__':
set_csv_weixin('D:/2020/新媒体监测/新媒体对比结果_微博_账号名称更改.csv')

64
新媒体更新脚本/更新抖音名称.py Normal file
View File

@ -0,0 +1,64 @@
# 更新新媒体脚本，结果为有出入的新媒体列表。
'''
1,将excel转为csv，使用utf-8编码集。
2、获取csv内容
3、与数据库中的数据逐项对比
'''
import csv
import re
import uuid
import psycopg2 as psycopg2
# code = None
# alias = None
# attention = None
# remark = None
# identificationcode = None
# function = None
# articleurl = None
# weixinid = None
# type = None
G2 = 'host=210.77.68.250 port=5432 dbname=newmediaDB3 user=newmedia password=newmedia2020!@#'
def set_csv_weixin(path):
with open(path, encoding='utf-8') as csvfile:
reader = csv.reader(csvfile)
for r in reader:
if r[1] == '删除':
print(r)
with psycopg2.connect(G2) as connection:
with connection.cursor() as cursor:
cursor.execute(
"delete from dashboard_douyin where code = '%s'" %(re.sub('\s+', '', r[0]).strip()))
connection.commit()
elif r[1] == '新增':
with open('D:/2020/新媒体监测/TASK_ALL.csv',encoding='utf-8') as c:
t = csv.reader(c)
for i in t:
print(i[4],r[0])
if re.sub('\s+', '', r[0]).strip() == re.sub('\s+', '', i[4]).strip():
with psycopg2.connect(G2) as connection:
with connection.cursor() as cursor:
try:
cursor.execute(
"select * from dashboard_organization where name = '%s'" % (
re.sub('\s+', '', i[1]).strip()))
connection.commit()
print(i[1] + '++++++++++++++++++++++++++++++++++++++++')
for c in cursor:
print(c[0] + "22222222222222222222222222222222222")
id = uuid.uuid4()
cursor.execute(
"insert into dashboard_douyin(id,code, created, updated,organization_id,image,alias,status,douyinid,attention, remark,identificationcode,function,articleurl) values (%s,%s,now(),now(),%s,'douyin.png','',3,%s,'','',%s,%s,%s)",
(str(id), re.sub('\s+', '', i[4]).strip(), c[0], i[13], i[3], i[8], i[12]))
connection.commit()
except Exception as e:
print(e)
if __name__ == '__main__':
set_csv_weixin('D:/2020/新媒体监测/新媒体对比结果_抖音_账号名称更改.csv')

60
新媒体更新脚本/查找其他名称更改.py Normal file
View File

@ -0,0 +1,60 @@
# 更新新媒体脚本，结果为有出入的新媒体列表。
'''
1,将excel转为csv，使用utf-8编码集。
2、获取csv内容
3、与数据库中的数据逐项对比
'''
import csv
import uuid
import psycopg2 as psycopg2
# code = None
# alias = None
# attention = None
# remark = None
# identificationcode = None
# function = None
# articleurl = None
# weixinid = None
# type = None
G2 = 'host=210.77.68.250 port=5432 dbname=newmediaDB3 user=newmedia password=newmedia2020!@#'
def get_csv_weixin(path):
with open("D:/2020/新媒体监测/新媒体对比结果_其他_账号名称更改.csv", "w", newline='',encoding='utf-8') as csvfile:
writer = csv.writer(csvfile)
qita_code_list = []
csv_code_list = []
r = None
with open(path, encoding='utf-8') as csvfile:
reader = csv.reader(csvfile)
for r in reader:
if r[1] != '单位全称':
if '微信' not in r[7] and '微博' not in r[7] and '头条' not in r[7] and '抖音' not in r[7]:
csv_code_list.append(r[4]+r[7])
with psycopg2.connect(G2) as connection:
with connection.cursor() as cursor:
try:
cursor.execute(
"select * from dashboard_qita")
connection.commit()
for c in cursor:
qita_code_list.append(c[1]+c[9])
except Exception as e:
print(e)
print(len(qita_code_list),len(csv_code_list))
print( len(list(set( qita_code_list ) ^ set( csv_code_list ))))
for i in list(set( qita_code_list ) ^ set( csv_code_list )):
if i in qita_code_list and i not in csv_code_list:
writer.writerow(
[i,'删除'])
elif i in csv_code_list and i not in qita_code_list:
writer.writerow(
[i, '新增'])
else:
writer.writerow(
[i, '异常'])
if __name__ == '__main__':
get_csv_weixin('D:/2020/新媒体监测/TASK_ALL.csv')

59
新媒体更新脚本/查找单位行政区划更改.py
View File

@ -0,0 +1,59 @@
# 更新新媒体脚本，结果为有出入的新媒体列表。
'''
1,将excel转为csv，使用utf-8编码集。
2、获取csv内容
3、与数据库中的数据逐项对比
'''
import csv
import uuid
import psycopg2 as psycopg2
# code = None
# alias = None
# attention = None
# remark = None
# identificationcode = None
# function = None
# articleurl = None
# weixinid = None
# type = None
G2 = 'host=210.77.68.250 port=5432 dbname=newmediaDB3 user=newmedia password=newmedia2020!@#'
def get_csv_weixin(path):
with open("D:/2020/新媒体监测/单位名称对比结果.csv", "w", newline='',encoding='utf-8') as csvfile:
writer = csv.writer(csvfile)
o_code_list = []
csv_code_list = []
with open(path, encoding='utf-8') as csvfile:
reader = csv.reader(csvfile)
for r in reader:
if r[1] != '单位全称':
csv_code_list.append(r[1])
with psycopg2.connect(G2) as connection:
with connection.cursor() as cursor:
try:
cursor.execute(
"select * from dashboard_organization")
connection.commit()
for c in cursor:
o_code_list.append(c[1])
except Exception as e:
print(e)
print(len(o_code_list),len(list(set(csv_code_list))))
print( len(list(set( o_code_list ) ^ set( csv_code_list ))))
for i in list(set( o_code_list ) ^ set( csv_code_list )):
if i in o_code_list and i not in csv_code_list:
writer.writerow(
[i,'删除'])
elif i in csv_code_list and i not in o_code_list:
writer.writerow(
[i, '新增'])
else:
writer.writerow(
[i, '异常'])
if __name__ == '__main__':
get_csv_weixin('D:/2020/新媒体监测/TASK_ALL.csv')

60
新媒体更新脚本/查找头条名称更改.py Normal file
View File

@ -0,0 +1,60 @@
# 更新新媒体脚本，结果为有出入的新媒体列表。
'''
1,将excel转为csv，使用utf-8编码集。
2、获取csv内容
3、与数据库中的数据逐项对比
'''
import csv
import uuid
import psycopg2 as psycopg2
# code = None
# alias = None
# attention = None
# remark = None
# identificationcode = None
# function = None
# articleurl = None
# weixinid = None
# type = None
G2 = 'host=210.77.68.250 port=5432 dbname=newmediaDB3 user=newmedia password=newmedia2020!@#'
def get_csv_weixin(path):
with open("D:/2020/新媒体监测/新媒体对比结果_头条_账号名称更改.csv", "w", newline='',encoding='utf-8') as csvfile:
writer = csv.writer(csvfile)
toutiao_code_list = []
csv_code_list = []
r = None
with open(path, encoding='utf-8') as csvfile:
reader = csv.reader(csvfile)
for r in reader:
if r[1] != '单位全称':
if '头条' in r[7]:
csv_code_list.append(r[4])
with psycopg2.connect(G2) as connection:
with connection.cursor() as cursor:
try:
cursor.execute(
"select * from dashboard_toutiao")
connection.commit()
for c in cursor:
toutiao_code_list.append(c[1])
except Exception as e:
print(e)
print(len(toutiao_code_list),len(csv_code_list))
print( len(list(set( toutiao_code_list ) ^ set( csv_code_list ))))
for i in list(set( toutiao_code_list ) ^ set( csv_code_list )):
if i in toutiao_code_list and i not in csv_code_list:
writer.writerow(
[i,'删除'])
elif i in csv_code_list and i not in toutiao_code_list:
writer.writerow(
[i, '新增'])
else:
writer.writerow(
[i, '异常'])
if __name__ == '__main__':
get_csv_weixin('D:/2020/新媒体监测/TASK_ALL.csv')

4
新媒体更新脚本/查找微信名称更改.py
View File

@ -24,16 +24,14 @@ G2 = 'host=210.77.68.250 port=5432 dbname=newmediaDB3 user=newmedia password=new
def get_csv_weixin(path):
with open("D:/2020/新媒体监测/新媒体对比结果_微信_账号名称更改.csv", "w", newline='',encoding='utf-8') as csvfile:
writer = csv.writer(csvfile)
writer.writerow(
["账号", "备注"])
weixin_code_list = []
csv_code_list = []
r = None
with open(path, encoding='utf-8') as csvfile:
reader = csv.reader(csvfile)
for r in reader:
if r[1] != '单位全称':
if '微信' in r[7]:
# print(r)
csv_code_list.append(r[4])
with psycopg2.connect(G2) as connection:
with connection.cursor() as cursor:

61
新媒体更新脚本/查找微博名称更改.py Normal file
View File

@ -0,0 +1,61 @@
# 更新新媒体脚本，结果为有出入的新媒体列表。
'''
1,将excel转为csv，使用utf-8编码集。
2、获取csv内容
3、与数据库中的数据逐项对比
'''
import csv
import uuid
import psycopg2 as psycopg2
# code = None
# alias = None
# attention = None
# remark = None
# identificationcode = None
# function = None
# articleurl = None
# weixinid = None
# type = None
G2 = 'host=210.77.68.250 port=5432 dbname=newmediaDB3 user=newmedia password=newmedia2020!@#'
def get_csv_weixin(path):
with open("D:/2020/新媒体监测/新媒体对比结果_微博_账号名称更改.csv", "w", newline='',encoding='utf-8') as csvfile:
writer = csv.writer(csvfile)
writer.writerow(
["账号", "备注"])
weibo_code_list = []
csv_code_list = []
with open(path, encoding='utf-8') as csvfile:
reader = csv.reader(csvfile)
for r in reader:
if r[1] != '单位全称':
if '微博' in r[7]:
csv_code_list.append(r[4])
with psycopg2.connect(G2) as connection:
with connection.cursor() as cursor:
try:
cursor.execute(
"select * from dashboard_weibo")
connection.commit()
for c in cursor:
weibo_code_list.append(c[1])
except Exception as e:
print(e)
print(len(weibo_code_list),len(csv_code_list))
print( len(list(set( weibo_code_list ) ^ set( csv_code_list ))))
for i in list(set( weibo_code_list ) ^ set( csv_code_list )):
if i in weibo_code_list and i not in csv_code_list:
writer.writerow(
[i,'删除'])
elif i in csv_code_list and i not in weibo_code_list:
writer.writerow(
[i, '新增'])
else:
writer.writerow(
[i, '异常'])
if __name__ == '__main__':
get_csv_weixin('D:/2020/新媒体监测/TASK_ALL.csv')

62
新媒体更新脚本/查找抖音名称更改.py Normal file
View File

@ -0,0 +1,62 @@
# 更新新媒体脚本，结果为有出入的新媒体列表。
'''
1,将excel转为csv，使用utf-8编码集。
2、获取csv内容
3、与数据库中的数据逐项对比
'''
import csv
import uuid
import psycopg2 as psycopg2
# code = None
# alias = None
# attention = None
# remark = None
# identificationcode = None
# function = None
# articleurl = None
# weixinid = None
# type = None
G2 = 'host=210.77.68.250 port=5432 dbname=newmediaDB3 user=newmedia password=newmedia2020!@#'
def get_csv_weixin(path):
with open("D:/2020/新媒体监测/新媒体对比结果_抖音_账号名称更改.csv", "w", newline='',encoding='utf-8') as csvfile:
writer = csv.writer(csvfile)
writer.writerow(
["账号", "备注"])
douyin_code_list = []
csv_code_list = []
r = None
with open(path, encoding='utf-8') as csvfile:
reader = csv.reader(csvfile)
for r in reader:
if r[1] != '单位全称':
if '抖音' in r[7]:
csv_code_list.append(r[4])
with psycopg2.connect(G2) as connection:
with connection.cursor() as cursor:
try:
cursor.execute(
"select * from dashboard_douyin")
connection.commit()
for c in cursor:
douyin_code_list.append(c[1])
except Exception as e:
print(e)
print(len(douyin_code_list),len(csv_code_list))
print( len(list(set( douyin_code_list ) ^ set( csv_code_list ))))
for i in list(set( douyin_code_list ) ^ set( csv_code_list )):
if i in douyin_code_list and i not in csv_code_list:
writer.writerow(
[i,'删除'])
elif i in csv_code_list and i not in douyin_code_list:
writer.writerow(
[i, '新增'])
else:
writer.writerow(
[i, '异常'])
if __name__ == '__main__':
get_csv_weixin('D:/2020/新媒体监测/TASK_ALL.csv')