! squid のLog の変換

 # tail /var/log/squid/access.log | perl -pe 's/\d+\.\d+/localtime $&/e' -  

----
http://wiki.squid-cache.org/Features/LogFormat

!Log check
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2014 Hiro.Ogawa
 
 '''
 Created on Oct 11, 2010
 
 @author: His Royal Highness
 
 The squid log reader will analyse the squid access.log file and produce stats on its
 content. it expects access.log to be in the same directory,
 but this can be changed to an absolute path as requried.
 '''
 
 ''' VARIABLES '''
 # filename = "./logs/access.log"
 # filename = "/home/squid/logs/access.log"
 filename = "/home/squid/logs/access.log.0"
 ## filename = "/home/squid/logs/access.log.1"
 file = open(filename)
 ''' Request Variables '''
 total_requests = 0
 total_cache_hits = 0
 total_cache_misses = 0
 ''' Byte Total Variables '''
 total_requested_bytes = 0
 total_cache_bytes = 0
 total_miss_bytes = 0
 total_other_bytes = 0
 host_requests = {}
 host_total_bytes = {}
 domain_requests = {}
 domain_bytes = {}
 
 ''' Strip back URL to what we will measure on '''
 def striptodomain(url):
    stripped = url.replace("http://", "")
    #stripped = stripped.repace("www.")
    bits = stripped.split('/')
    hostname = bits[0]
    return hostname
 
 
 ''' Rip through the file and generate the stats '''
 for line in file:
    bit = line.split()
    if len(bit) < 9:
        print len(bit)
        continue
    total_requests += 1
 
    ''' Add in the host request and bytes totals '''
    ## print bit[2],bit[4]
    if bit[2] in host_requests:
        host_requests[bit[2]] += 1
        host_total_bytes[bit[2]] = int(host_total_bytes[bit[2]]) + int(bit[4])
    else:
        host_requests[bit[2]] = 1
        host_total_bytes[bit[2]] = int(bit[4])
 
 
    ''' Add in the Domain request and bytes totals '''
    hostname = striptodomain(bit[6])
    if hostname in domain_requests:
        domain_requests[hostname] += 1
        domain_bytes[hostname] = int(domain_bytes[hostname]) + int(bit[4])
    else:
        domain_requests[hostname] = 1
        domain_bytes[hostname] = int(bit[4])
 
 
 
    if 'TCP_HIT' in line or 'TCP_MEM_HIT' in line or 'TCP_NEGATIVE_HIT' in line or 'TCP_IMS_HIT' in line or 'TCP_REFRESH_HIT' in line:
        total_cache_hits += 1
        total_cache_bytes += int(bit[4])
        total_requested_bytes += int(bit[4])
 
    elif "TCP_MISS" in line or 'TCP_REFRESH_MISS' in line or 'TCP_CLIENT_REFRESH_MISS' in line:
        total_cache_misses += 1
        total_miss_bytes += int(bit[4])
        total_requested_bytes += int(bit[4])
 
    else:
        total_other_bytes += int(bit[4])
        total_requested_bytes += int(bit[4])
 file.close()
 
 
 ''' Extra file stats '''
 total_other = (total_requests - total_cache_hits) - total_cache_misses
 
 
 ''' Calculate Percent values '''
 total_cache_hits_pc = round((1.0 * total_cache_hits / total_requests) * 100,2)
 total_cache_misses_pc = round((1.0 * total_cache_misses / total_requests) * 100,2)
 total_other_pc = round((1.0 * total_other / total_requests) * 100,2)
 
 total_cache_bytes_pc = round((1.0*total_cache_bytes/total_requested_bytes) * 100,2)
 total_miss_bytes_pc = round((1.0 *total_miss_bytes/ total_requested_bytes) * 100,2)
 total_other_bytes_pc = round((1.0 *total_other_bytes/total_requested_bytes)* 100,2)
 
 
 
 ''' File type counts'''
 print " "
 print "Caching Performance"
 print "Total requests: \t\t", total_requests
 print "Total Cache hits:\t\t",total_cache_hits, total_cache_hits_pc, "%"
 print "Total Cache misses:\t" ,total_cache_misses, total_cache_misses_pc, "%"
 print "Total Other:\t\t" ,total_other, total_other_pc, "%"
 print ""
 print "Total bytes: \t\t", total_requested_bytes
 print "Total Cache bytes:\t",total_cache_bytes, total_cache_bytes_pc, "%"
 print "Total Miss bytes:\t\t",total_miss_bytes, total_miss_bytes_pc, "%"
 print "Total Other bytes:\t",total_other_bytes, total_other_bytes_pc, "%"
 print ""
 print ""
 print ""
 
 
 
 
 ''' Print Domain statistics '''
 #sorted_domain_requests = sorted(domain_requests)
 print "Most Requested Domains (hits)"
 temp_list = domain_requests.keys()
 temp_list.sort( key = domain_requests.__getitem__, reverse = True)
 for a in range(25):
    print domain_requests[temp_list[a]]  , "\t\t" , temp_list[a]
 print ""
 print ""
 
 
 print "Highest Data Transfer by Domain (bytes)"
 temp_list = domain_bytes.keys()
 temp_list.sort( key = domain_bytes.__getitem__, reverse = True)
 for a in range(25):
    print domain_bytes[temp_list[a]]  , "\t" , temp_list[a]
 print ""
 print ""
 
 print "Most requesting hosts (hits)"
 temp_list = host_requests.keys()
 temp_list.sort( key = host_requests.__getitem__, reverse = True)
 if len(temp_list) < 25:
    hosts = len(temp_list)
 else:
    hosts = 25
 for a in range(hosts):
    print host_requests[temp_list[a]]  , "\t" , temp_list[a]
 print ""
 print ""
 
 print "Highest Data Transfer by Host (bytes)"
 temp_list = host_total_bytes.keys()
 temp_list.sort( key = host_total_bytes.__getitem__, reverse = True)
 for a in range(hosts):
    print host_total_bytes[temp_list[a]]  , "\t" , temp_list[a]
 print ""
 print ""



----
!Log check2
green@vbfb1:~/Yama1 % cat ./q2.py
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2014 Hiro.Ogawa
 
 import sqlite3
 
 class DDvar:
    def __init__(self):
        self.axisx = {}
        self.axisy = {}
        self.cnt_axisx = 0
        self.cnt_axisy = 0
        self.imat = [[0 for i in range(100)] for j in range(100)]
 
 
    def strmat(self,ax,ay):
            retx = 0
            rety = 0
            if ax in self.axisx:
                retx = self.axisx[ax]
            else:
                self.cnt_axisx = self.cnt_axisx + 1
                self.axisx[ax] = self.cnt_axisx
                retx = self.cnt_axisx
 
            if ay in self.axisy:
                rety = self.axisy[ay]
            else:
                self.cnt_axisy = self.cnt_axisy + 1
                self.axisy[ay] = self.cnt_axisy
                rety = self.cnt_axisy
 
            return(retx,rety)
 
 
    def setimat(self,ax,ay,data):
            ( x,y ) = self.strmat( ax, ay )
            self.imat[x][y] = data
 
    def getimat(self,ax,ay):
            ( x,y ) = self.strmat( ax, ay )
            rdat = self.imat[x][y]
            return rdat
 
 
 if __name__ == "__main__":
    axi = DDvar()
    count = 0
 
    for i in [ 'a' , 'b', 'c', 'd' ]:
        for j in [ '11', '22', '33', '44']:
            ## print i,j
            ## imat[i][j]=count
            axi.setimat( i , j , count )
            count = count + 1
 
 
 
    for ii in [ 'a' , 'b', 'c', 'd' ]:
        for jj in [ '11', '22', '33', '44']:
            print ii,jj,axi.getimat( ii, jj )

----
! elapsed time check
 #!/usr/bin/env python
 # coding: utf-8
 
 import os
 
 fname="./logs/access.log"
 
 fp = open(fname)
 
 rdata = fp.read()
 
 fp.close()
 
 lines = rdata.split('\n')
 
 for line in lines :
    sp3 = line.split()[1]
    ## spp3 = sp3.encode('utf_8')
    if sp3.isdigit :
        ss = int(sp3)
        if ss > 1000 :
            sp1 = float( line.split()[0] )
            print datetime.datetime.fromtimestamp(sp1),
            print line
    # print sp3

----
! ログ時間の
 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
 
 import time
 import datetime
 
 fname = "./180122/access.log.0"
 
 fp = open(fname)
 
 stime = time.mktime(datetime.datetime(2018, 1, 19, 0, 0, 0, 1).timetuple())
 etime = time.mktime(datetime.datetime(2018, 1, 19, 23, 59, 59, 9).timetuple())
 
 print (stime )
 
 for fline in fp :
    # print ( fline.split()[0] )
    ti = fline.split()[0]
    lips = fline.split()[2]
    lurl = fline.split()[6]
    # print ( ips )
    ltime = float( ti )
    ## print ( time.localtime(ltime)) 
    if ( (ltime > stime)  & ( ltime < etime )):
        s2 = int(lips.split(".")[2])
        s3 = int(lips.split(".")[3])
        if ( ( s2 == 3) & ((s3 > 102 ) & (s3 < 120))): 
            print ( datetime.datetime.fromtimestamp(ltime),end="") 
            print ( ","+lips + "," + lurl)
----