1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
|
import os
import stat
import subprocess
import md5
import time
import exceptions
indexdir="/mnt/gluster/testdir/"
inputdirs=[
"/mnt/gluster/input1/",
"/mnt/gluster/input2/",
"/mnt/gluster/input3/",
"/mnt/gluster/input4/",
"/mnt/gluster/input5/"]
def getstat(filepath):
return os.stat(filepath)
def getmd5sum(path):
f = open(path,'rb')
m = md5.new()
while True:
data = f.read(8096)
if(not data):
break
m.update(data)
f.close()
return m.hexdigest()
def listdir(path):
cmd = "ls -rt " + path
filelist = []
process = subprocess.Popen(cmd, shell=True,stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
for line in process.stdout:
#print line
line = str(line).rstrip()
filelist.append(path + str(line))
output,error = process.communicate()
#print output
return filelist
def writeindex(indexfilepath,map):
tempfile = indexfilepath + ".tmp"
fd = open(tempfile, 'w')
for fname in map.keys():
lst = map[fname]
print >> fd , '%s %s' % (lst[0], lst[1])
fd.flush()
fd.close()
os.rename(tempfile, indexfilepath)
def loadindex(indexfilepath):
try:
f = open(indexfilepath,'r')
lst = []
for line in f:
lst.append(line)
f.close()
except Exception,e:
print e,indexfilepath
while(True):
for dir in inputdirs:
map = {}
ret = listdir(dir)
fname = dir.split("/")[-2]
idxname = fname + ".idx"
print "dir = " + str(dir)
for x in ret:
sts = getstat(x)
m5 = getmd5sum(x)
lst = [m5,sts]
map[x] = lst
if os.path.exists(indexdir + "/" + idxname):
loadindex(indexdir + "/" + idxname)
writeindex(indexdir + "/" + idxname,map)
met = listdir(indexdir)
for z in met:
loadindex(z)
mapx={}
metname = indexdir + "/meta.idx"
for y in met:
sts = getstat(y)
m5 = getmd5sum(y)
lst = [m5,sts]
mapx[y] = lst
writeindex(metname,mapx)
print "sleeping for 60 secs"
time.sleep(10)
#listdir(inputdirs[0])
|