/[gentoo-src]/bittorrent/bt_daemon.py
Gentoo

Diff of /bittorrent/bt_daemon.py

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

Revision 1.1 Revision 1.2
… …
2 2
3# Greatly modified by carpaski@gentoo.org 3# Greatly modified by carpaski@gentoo.org
4# btdownloadheadless written by Bram Cohen 4# btdownloadheadless written by Bram Cohen
5# see LICENSE.txt for license information 5# see LICENSE.txt for license information
6 6
7import os
8os.nice(5)
7 9
8# This is 2^x ... put the x below -- Chunk size for sha1 -- 18 == 256k 10# This is 2^x ... put the x below -- Chunk size for sha1 -- 18 == 256k
9piece_size_pow2 = 18 11piece_size_pow2 = 18
10delay = 0 12MAX_STARTING_THREADS = 15
13LOOP_MOD = 4320 # Rotating value (for logs)
14ROTATE_LOGS = 0 # Make one log or roate into loop_mod logs
15cycle_sleep = 20
16spawn_thread_delay = 0
11debug = 0 17debug = 0
12 18
13# No trailing / 19# No trailing /
14file_base = "/usr/portage/distfiles" 20file_base = "/usr/portage/distfiles"
15torrent_base = "/tmp/torrents" 21torrent_base = "/tmp/torrents"
16 22
17COMMENT = "Gentoo Linux BitTorrent Mirror System" 23COMMENT = "Gentoo Linux BitTorrent Mirror System"
18tracker = "http://egret.gentoo.org:6969/announce" 24tracker = "http://egret.gentoo.org:6969/announce"
19 25
20# Useless variables 26# Useless variables
21cols = 80 27cols = 80
28
29
22import BitTorrent 30import BitTorrent
23import os
24from BitTorrent.download import download 31from BitTorrent.download import download
25from threading import Event,Thread 32from threading import Event,Thread
26import types 33import types
27import re 34import re
28from os import listdir, getcwd, stat 35from os import listdir, getcwd, stat
… …
33from time import time,ctime,sleep 40from time import time,ctime,sleep
34 41
35from BitTorrent.makemetafile import make_meta_file 42from BitTorrent.makemetafile import make_meta_file
36 43
37def hours(n): 44def hours(n):
38 if n == -1: 45 if n == -1:
39 return '<unknown>' 46 return '<unknown>'
40 if n == 0: 47 if n == 0:
41 return 'complete!' 48 return 'complete!'
42 n = long(n) 49 n = long(n)
43 h, r = divmod(n, 60 * 60) 50 h, r = divmod(n, 60 * 60)
44 m, sec = divmod(r, 60) 51 m, sec = divmod(r, 60)
45 if h > 1000000: 52 if h > 1000000:
46 return '<unknown>' 53 return '<unknown>'
47 if h > 0: 54 if h > 0:
48 return '%d hour %02d min %02d sec' % (h, m, sec) 55 return '%d hour %02d min %02d sec' % (h, m, sec)
49 else: 56 else:
50 return '%d min %02d sec' % (m, sec) 57 return '%d min %02d sec' % (m, sec)
51 58
52class HeadlessDisplayer: 59class HeadlessDisplayer:
53 def __init__(self): 60 def __init__(self):
54 self.done = False 61 self.done = False
55 self.file = '' 62 self.failed = False
56 self.percentDone = '' 63 self.file = ''
57 self.timeEst = '' 64 self.activity = ''
65 self.percentDone = 0.0
66 self.timeEst = 0
58 self.downloadTo = '' 67 self.downloadTo = ''
59 self.downRate = '' 68 self.downRate = 0.0
60 self.upRate = '' 69 self.upRate = 0.0
61 self.downTotal = '' 70 self.downTotal = 0.0
62 self.upTotal = '' 71 self.upTotal = 0
63 self.errors = [] 72 self.errors = []
64 self.last_update_time = 0 73 self.last_update_time = 0
65 74
66 def finished(self): 75 def finished(self):
67 self.done = True 76 self.done = True
68 self.percentDone = '100' 77 self.percentDone = 100
69 self.timeEst = 'Download Succeeded!' 78 self.timeEst = 0
70 self.downRate = '' 79 self.downRate = 0
71 self.display({}) 80 self.display({})
72 81
73 def failed(self): 82 def failed(self):
74 self.done = True 83 self.done = True
84 self.failed = True
75 self.percentDone = '0' 85 self.percentDone = 0
76 self.timeEst = 'Download Failed!' 86 self.timeEst = -1
77 self.downRate = '' 87 self.downRate = 0
78 self.display({}) 88 self.display({})
79 89
80 def error(self, errormsg): 90 def error(self, errormsg):
81 self.errors.append(errormsg) 91 self.errors.append(errormsg)
82 self.display({}) 92 self.display({})
83 93
84 def display(self, dict): 94 def display(self, dict):
95 if ((self.last_update_time + 1) > time()):
96 if dict.get('fractionDone') not in (0.0, 1.0):
97 if not dict.has_key('activity'):
98 return
99 self.last_update_time = time()
100 if dict.has_key('fractionDone'):
101 self.percentDone = float(int(dict['fractionDone'] * 1000)) / 10
102 if dict.has_key('timeEst'):
103 self.timeEst = long(dict['timeEst'])
104 if dict.has_key('activity') and not self.done:
105 self.activity = dict['activity']
106 if dict.has_key('downRate'):
107 self.downRate = float(dict['downRate']) / (1 << 10)
108 if dict.has_key('upRate'):
109 self.upRate = float(dict['upRate']) / (1 << 10)
110 if dict.has_key('upTotal'):
111 self.upTotal = dict['upTotal']
112 if dict.has_key('downTotal'):
113 self.downTotal = dict['downTotal']
85 for x in self.errors: 114 for x in self.errors:
86 print self.file+": "+str(x) 115 print self.file+": "+str(x)
87 self.errors = [] 116 self.errors = []
88 return 117 return
89 118
90 def chooseFile(self, default, size, saveas, dir): 119 def chooseFile(self, default, size, saveas, dir):
91 self.file = '%s (%.1f MB)' % (default, float(size) / (1 << 20)) 120 self.file = '%s (%.1f MB)' % (default, float(size) / (1 << 20))
92 if saveas != '': 121 if saveas != '':
93 default = saveas 122 default = saveas
94 self.downloadTo = abspath(default) 123 self.downloadTo = abspath(default)
95 return default 124 return default
96 125
97 def newpath(self, path): 126 def newpath(self, path):
98 self.downloadTo = path 127 self.downloadTo = path
99 128
100def prefix_array(array,prefix,doblanks=1): 129def prefix_array(array,prefix,doblanks=1):
101 """Prepends a given prefix to each element in an Array/List/Tuple. 130 """Prepends a given prefix to each element in an Array/List/Tuple.
102 Returns a List.""" 131 Returns a List."""
103 if type(array) not in [types.ListType, types.TupleType]: 132 if type(array) not in [types.ListType, types.TupleType]:
104 raise TypeError, "List or Tuple expected. Got %s" % type(array) 133 raise TypeError, "List or Tuple expected. Got %s" % type(array)
105 newarray=[] 134 newarray=[]
106 for x in array: 135 for x in array:
107 if x or doblanks: 136 if x or doblanks:
108 newarray.append(prefix + x) 137 newarray.append(prefix + x)
109 else: 138 else:
110 newarray.append(x) 139 newarray.append(x)
111 return newarray 140 return newarray
112 141
113 142
114def list_files(dirlist): 143def list_files(dirlist):
115 file_list = [] 144 file_list = []
116 for mydir in dirlist: 145 for mydir in dirlist:
… …
129def progress(amount): 158def progress(amount):
130 # Dummy function for the create-torrent progress meter 159 # Dummy function for the create-torrent progress meter
131 pass 160 pass
132 161
133 162
163def writemsg(msg,fileName,overwrite=0, create=0):
164 if overwrite or create:
165 fileHandle = open(fileName, "w")
166 else:
167 fileHandle = open(fileName, "a")
134 168
169 fileHandle.write(msg+"\n")
170 fileHandle.flush()
171 fileHandle.close()
172 print msg
135 173
136if __name__ == '__main__': 174if __name__ == '__main__':
137 175
138 threads = {} 176 threads = {}
139 dirs = [torrent_base[:],file_base[:]] 177 dirs = [torrent_base[:],file_base[:]]
140 re_file = re.compile("("+torrent_base[:]+"|"+file_base[:]+")(/.+?)(\.torrent)?$") 178 re_file = re.compile("("+torrent_base[:]+"|"+file_base[:]+")(/.+?)(\.torrent)?$")
179 absmax_upRate = 0.0
180 absmax_upFile = ''
141 181
182 loop_val = 0
142 while(True): 183 while(True):
184 loop_val = (loop_val + 1) % LOOP_MOD
143 files = list_files(dirs) 185 files = list_files(dirs)
144 files.sort() 186 files.sort()
145 new_files = [] 187 new_files = []
146 torrents = [] 188 torrents = []
147 189
… …
166 print "New file:",f 208 print "New file:",f
167 new_files.append(match.group(2)) 209 new_files.append(match.group(2))
168 210
169 for t in threads.keys(): 211 for t in threads.keys():
170 # Remove threads that we don't have a bittorrent for. 212 # Remove threads that we don't have a bittorrent for.
171 if t not in torrents: 213 if t not in torrents or not os.path.exists(file_base+t):
172 print "Stopping torrent [%s]: %s" % (ctime(),t) 214 print "Stopping torrent [%s]: %s" % (ctime(),t)
173 threads[t][0]._Thread__stop() 215 threads[t][0]._Thread__stop()
174 del threads[t] 216 del threads[t]
175 while(t in torrents): 217 while(t in torrents):
176 del torrents[torrents.index(t)] 218 del torrents[torrents.index(t)]
177 219
178 elif not threads[t][0].isAlive(): 220 elif not threads[t][0].isAlive():
179 print "Thread died [%s]: %s" % (ctime(),t) 221 print "Thread died [%s]: %s" % (ctime(),t)
180 del threads[t] 222 del threads[t]
223
224 fetching = 0
225 completed = 0
226 stalled = 0
227 totalup = 0
228 rateup = 0
229 ratedown = 0
230
231 max_up_rate = 0
232 max_up_file = ''
233 max_down_rate = 0
234 max_down_file = ''
235 for t in threads.keys():
236 # Let's do some stats.
237 if not threads[t][1].failed and threads[t][0].isAlive():
238 if threads[t][1].done:
239 completed += 1
240 else:
241 fetching += 1
242 if threads[t][1].downRate == 0:
243 stalled += 1
244
245 totalup += threads[t][1].upTotal
246
247 rateup += threads[t][1].upRate
248 if threads[t][1].upRate > max_up_rate:
249 max_up_rate = threads[t][1].upRate
250 max_up_file = threads[t][1].file[:]
251 if threads[t][1].upRate > absmax_upRate:
252 absmax_upRate = threads[t][1].upRate
253 absmax_upFile = threads[t][1].file[:]
254
255 ratedown += threads[t][1].downRate
256 if threads[t][1].downRate > max_down_rate:
257 max_down_rate = threads[t][1].downRate
258 max_down_file = threads[t][1].file[:]
259
260 if os.path.exists(".stats"):
261 if ROTATE_LOGS:
262 filename = "bt_daemon-%d-%4d.stats" % (os.getpid(),loop_val)
263 makenew = 1
264 else:
265 filename = "bt_daemon-%d.stats" % (os.getpid())
266 makenew = 0
267 writemsg("",filename,overwrite=makenew)
268 writemsg("Stats: %s" % (ctime()) ,filename)
269 writemsg("Seeding: %d" % (completed) ,filename)
270 writemsg("Fetching: %d(%d)" % (fetching,stalled) ,filename)
271 writemsg("Total up: %.3f MB" % (totalup) ,filename)
272 writemsg("UpRate: %.2f k/s" % (rateup) ,filename)
273 writemsg("UpMaxNow: %.2f k/s '%s'" % (max_up_rate,max_up_file) ,filename)
274 writemsg("UpAbsMax: %.2f k/s" % (absmax_upRate) ,filename)
275 writemsg("DownRate: %.2f k/s" % (ratedown) ,filename)
276 writemsg("DownBest: %.2f k/s '%s'" % (max_down_rate,max_down_file) ,filename)
277 writemsg("",filename)
181 278
182 for t in torrents: 279 for t in torrents:
183 # Start threads up. 280 # Start threads up if we have load free for them. Startup is harsh.
184 if t not in threads.keys(): 281 if t not in threads.keys() and ((len(threads.keys())-completed) < MAX_STARTING_THREADS):
185 print "Starting torrent [%s]: %s" % (ctime(),t) 282 print "Starting torrent [%s]: %s" % (ctime(),t)
186 h = HeadlessDisplayer() 283 h = HeadlessDisplayer()
187 try: 284 try:
188 if not os.path.isdir(file_base+"/"+os.path.dirname(t)): 285 if not os.path.isdir(file_base+"/"+os.path.dirname(t)):
189 os.makedirs(file_base+"/"+os.path.dirname(t)) 286 os.makedirs(file_base+"/"+os.path.dirname(t))
190 t_thread = Thread(target=download,name="BitTorrent::"+t,args=(['--responsefile', torrent_base+"/"+t+".torrent", '--saveas', file_base+"/"+t], h.chooseFile, h.display, h.finished, h.error, Event(), cols)) 287 t_thread = Thread(target=download,name="BitTorrent::"+t,args=(['--responsefile', torrent_base+"/"+t+".torrent", '--saveas', file_base+"/"+t], h.chooseFile, h.display, h.finished, h.error, Event(), cols))
191 threads[t] = [t_thread,h] 288 threads[t] = [t_thread,h]
192 t_thread.start() 289 t_thread.start()
193 sleep(delay) 290 sleep(spawn_thread_delay)
194 except Exception, e: 291 except Exception, e:
195 print "Failed to start thread: %s" % (t) 292 print "Failed to start thread: %s" % (t)
196 print "Reason: %s" % (str(e)) 293 print "Reason: %s" % (str(e))
197 294
198 for f in new_files: 295 for f in new_files:
… …
203 os.makedirs(torrent_base+"/"+os.path.dirname(f)) 300 os.makedirs(torrent_base+"/"+os.path.dirname(f))
204 make_meta_file(file_base+"/"+f,tracker,piece_size_pow2,progress=progress,comment=COMMENT,target=torrent_base+"/"+f+".torrent") 301 make_meta_file(file_base+"/"+f,tracker,piece_size_pow2,progress=progress,comment=COMMENT,target=torrent_base+"/"+f+".torrent")
205 except Exception, e: 302 except Exception, e:
206 print "Failed to create torrent: "+str(e) 303 print "Failed to create torrent: "+str(e)
207 304
208 sleep(1) 305 sleep(cycle_sleep)
Legend:
Removed from v.1.1  
changed lines
  Added in v.1.2
  ViewVC Help
Powered by ViewVC 1.1.13