bob
/
weblog-import
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix access problem, mv tmp file to /tmp

This commit is contained in:
wlx 2016-05-20 16:49:21 +08:00
parent dba1b9960b
commit d4d2508d99
2 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
log.sql
View File

@ -1,5 +1,5 @@
delete from tmplog;
copy tmplog (log) from '/root/westdclog/westdc-step2.log';
copy tmplog (log) from '/tmp/westdc-step2.log';
insert into accesslog1 (datetime,ip,method,path,size,uuid)
select t.datetime,t.ip::inet,t.method,t.path,t.size,t.u[1]::uuid from
(

6
westdclog.sh
View File

@ -2,9 +2,13 @@
cd /root/westdclog/
rm westdc.log*
rm westdc-step*
mergelog /var/log/apache2/card.westgis.ac.cn-access-`date --date="1 days ago" +%Y%m%d`.log* /var/log/apache2/heihedata.org-access-`date --date="1 days ago" +%Y%m%d`.log* /var/log/apache2/westdc.westgis.ac.cn-access-`date --date="1 days ago" +%Y%m%d`.log* > westdc.log
mergelog /var/log/apache2/card.westgis.ac.cn-access-`date --date="1 days ago" +%Y%m%d`.log* /var/log/apache2/www.heihedata.org-access-`date --date="1 days ago" +%Y%m%d`.log* /var/log/apache2/westdc.westgis.ac.cn-access-`date --date="1 days ago" +%Y%m%d`.log* > westdc.log
./urldecode.sh
cat westdc.log.new | grep "/data\|/water\|/hiwater\|/haihe\|/heihe\|/search\|/service/attach" | grep -v "/js/\|/css/\|/images/\|/xml/\|/thumb/\|/feed\|/account\|/knowledge\|/doc/\|/admin/\|metainfo/\|filelist/\|/gndata/\|HEAD\|/related\|/vdimg\|/imgcode" | grep -v "bot/\|spider\|msnbot\|godtroop" | grep -v "Sosospider\|Sogou-Spider\|Sogou web spider\|Baiduspider\|bingbot/\|Googlebot\|Feedfetcher-Google\|Yahoo! Slurp\|MJ12bot/\|Exabot/\|ia_archiver\|TurnitinBot/\|ichiro/\|msnbot/\|psbot/0\|psbot/0\|msnbot-media/\|Bot\|bot\|EC2Link\|P.Arthur\|crawler\|Tianwang\|Spider\|findlinks\|Crawler" | grep " 200 " | grep -v "118.192.35.56\|210.77.68.215\|210.77.68.243\|210.77.68.248\|210.77.69.73\|210.72.80.163\|210.72.81.52\|210.77.68.241\|210.77.68.252\|2001:cc0" > westdc-step0.log
cat westdc-step0.log | grep "/data/view/uuid/[0-9a-fA-F]\{8\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{12\} \|/heihe/view/uuid/[0-9a-fA-F]\{8\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{12\} \|/water/view/uuid/[0-9a-fA-F]\{8\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{12\} \|/hiwater/view/uuid/[0-9a-fA-F]\{8\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{12\} \|/haihe/view/uuid/[0-9a-fA-F]\{8\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{12\} \|/data/[0-9a-fA-F]\{8\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{12\} \|/water/[0-9a-fA-F]\{8\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{12\} \|/heihe/[0-9a-fA-F]\{8\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{12\} \|/hiwater/[0-9a-fA-F]\{8\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{12\} \|/haihe/[0-9a-fA-F]\{8\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{4\}-[0-9a-fA-F]\{12\}" > westdc-step1.log
awk '$7 != "/" {print $4,$1,$6,$7,$10}' westdc-step1.log > westdc-step2.log
cp westdc-step2.log /tmp/
cp log.sql /tmp/
su -c "psql metadata -f log.sql" postgres
rm /tmp/westdc-step2.log
rm /tmp/log.sql