Last modified 3 years ago
Last modified on 07/02/13 14:45:56
Following is the script we are using to import PuffinServer Nginx logs into Piwik, see PiwikServer#ServerLogAnalytics
#!/bin/bash # date format used in log files names DATE=$(date "+%Y-%m-%d") # location of the latest log file NEW_LOG="/home/puffin/nginx/puffin-nginx-$DATE.log" LOGFILE="/home/puffin/nginx/puffin-nginx.log" # check the log file exists and then fix it # and import it to piwik if [[ -f $NEW_LOG ]]; then # fix the issue with the proxy ip address being added to the first field cat $NEW_LOG | sed 's/, 127.0.0.1//' > $LOGFILE # import the fixed log file python /web/stats.transitionnetwork.org/piwik/misc/log-analytics/import_logs.py \ --url=https://stats.transitionnetwork.org/ \ --idsite=12 --enable-static --enable-bots --enable-http-errors --enable-http-redirect \ --log-format-regex='"(?P<ip>\S+)" (?P<host>\S+) \[(?P<date>.*?) (?P<timezone>.*?)\] (?P<status>\S+) (?P<length>\S+) \S+ \S+ "(?P<referrer>.*?)" "(?P<user_agent>.*?)" \S+ "\S+"' \ --recorders=8 $LOGFILE fi # delete the log file rm $NEW_LOG