/[gentoo-src]/bittorrent/bt_daemon.py
Gentoo

Contents of /bittorrent/bt_daemon.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.5 - (show annotations) (download) (as text)
Wed Jun 16 05:15:31 2004 UTC (10 years, 4 months ago) by carpaski
Branch: MAIN
Changes since 1.4: +297 -220 lines
File MIME type: text/x-python
Updates to config file and descriptions.

1 #!/usr/bin/python -O
2 # $Id: bt_daemon.py,v 1.4 2004/05/25 23:51:44 carpaski Exp $
3 #
4 # Greatly modified by carpaski@gentoo.org
5 # btdownloadheadless written by Bram Cohen
6 # see LICENSE.txt for license information
7
8 import os,sys,string
9 os.nice(5)
10
11 sys.path = ["/usr/lib/portage/pym"] + sys.path + ["/usr/bin"]
12 from portage import getconfig
13
14 import BitTorrent
15 from BitTorrent.download import download
16
17 from threading import Event,Thread
18 import types,traceback,re,random
19
20 from os import listdir, getcwd, stat
21 from stat import ST_SIZE,ST_MTIME
22 from os.path import abspath,isdir,isfile,normpath
23 from sys import argv, stdout, path
24 from cStringIO import StringIO
25 from time import time,ctime,sleep
26
27 from btmakemetafile import make_meta_file
28
29 def hours(n):
30 if n == -1:
31 return '<unknown>'
32 if n == 0:
33 return 'complete!'
34 n = long(n)
35 h, r = divmod(n, 60 * 60)
36 m, sec = divmod(r, 60)
37 if h > 1000000:
38 return '<unknown>'
39 if h > 0:
40 return '%d hour %02d min %02d sec' % (h, m, sec)
41 else:
42 return '%d min %02d sec' % (m, sec)
43
44 class HeadlessDisplayer:
45 def __init__(self):
46 self.active = False
47 self.done = False
48 self.failed = False
49 self.file = ''
50 self.activity = ''
51 self.percentDone = 0.0
52 self.timeEst = 0
53 self.downloadTo = ''
54 self.downRate = 0.0
55 self.upRate = 0.0
56 self.downTotal = 0.0
57 self.upTotal = 0
58 self.errors = []
59 self.last_update_time = 0
60
61 def finished(self):
62 self.done = True
63 self.percentDone = 100
64 self.timeEst = 0
65 self.downRate = 0
66 self.display({})
67
68 def failed(self):
69 self.done = True
70 self.failed = True
71 self.percentDone = 0
72 self.timeEst = -1
73 self.downRate = 0
74 self.display({})
75
76 def error(self, errormsg):
77 self.errors.append(errormsg)
78 self.display({})
79
80 def display(self, dict):
81 if ((self.last_update_time + 1) > time()):
82 if dict.get('fractionDone') not in (0.0, 1.0):
83 if not dict.has_key('activity'):
84 return
85 self.last_update_time = time()
86 if dict.has_key('fractionDone'):
87 self.percentDone = float(int(dict['fractionDone'] * 1000)) / 10
88 if dict.has_key('timeEst'):
89 self.timeEst = long(dict['timeEst'])
90 if dict.has_key('activity') and not self.done:
91 self.activity = dict['activity']
92 if dict.has_key('downRate'):
93 self.downRate = float(dict['downRate']) / (1 << 10)
94 if self.downRate > 0:
95 self.has_transfered = True
96 if dict.has_key('upRate'):
97 self.upRate = float(dict['upRate']) / (1 << 10)
98 if self.upRate > 0:
99 self.has_transfered = True
100 if dict.has_key('upTotal'):
101 self.upTotal = dict['upTotal']
102 if dict.has_key('downTotal'):
103 self.active = True
104 self.downTotal = dict['downTotal']
105 for x in self.errors:
106 print self.file+": "+str(x)
107 self.errors = []
108 return
109
110 def chooseFile(self, default, size, saveas, dir):
111 self.file = '%s (%.1f MB)' % (default, float(size) / (1 << 20))
112 if saveas != '':
113 default = saveas
114 self.downloadTo = abspath(default)
115 return default
116
117 def newpath(self, path):
118 self.downloadTo = path
119
120 def prefix_array(array,prefix,doblanks=1):
121 """Prepends a given prefix to each element in an Array/List/Tuple.
122 Returns a List."""
123 if type(array) not in [types.ListType, types.TupleType]:
124 raise TypeError, "List or Tuple expected. Got %s" % type(array)
125 newarray=[]
126 for x in array:
127 if x or doblanks:
128 newarray.append(prefix + x)
129 else:
130 newarray.append(x)
131 return newarray
132
133 def list_files(dirlist):
134 file_list = []
135 for mydir in dirlist:
136 if isdir(mydir):
137 try:
138 file_list += list_files(prefix_array(listdir(mydir),mydir+"/"))
139 except Exception,e:
140 print e
141 elif isfile(mydir):
142 file_list.append(mydir)
143 else:
144 print "Unknown type:",mydir
145 return file_list
146
147
148 def progress(amount):
149 # Dummy function for the create-torrent progress meter
150 pass
151
152
153 def writemsg(msg,fileName,overwrite=0,create=0,echo=1):
154 if overwrite or create:
155 fileHandle = open(fileName, "w")
156 else:
157 fileHandle = open(fileName, "a")
158
159 fileHandle.write(msg+"\n")
160 fileHandle.flush()
161 fileHandle.close()
162 if echo:
163 print msg
164
165
166 # Useless variables
167 cols = 80
168
169 # This is 2^x ... put the x below -- Chunk size for sha1 -- 18 == 256k
170 piece_size_pow2 = 18
171 max_starting_threads = 1
172 loop_mod = 4320 # Rotating value (for logs)
173 rotate_logs = 0 # Make one log or roate into loop_mod logs
174 cycle_sleep = 20
175 spawn_thread_delay = 0
176 debug = 0
177 minimum_age = 30
178 port_min = 6881
179 port_max = 7280
180 check_hashes = 0
181 shuffle_files = 1
182 master_seed = 0
183 min_uploads = 20
184 max_uploads = 20
185 download_slice_size = 65536
186 max_slice_length = 131072
187 request_backlog = 20
188 timeout = 90
189 timeout_check_interval = 20
190 stats = 1
191
192 # No trailing /
193 file_base = "/usr/portage/distfiles"
194 torrent_base = "/tmp/torrents"
195
196 comment = "Gentoo Linux BitTorrent Mirror System"
197 tracker = "http://egret.gentoo.org:6969/announce"
198
199 if __name__ == '__main__':
200 try:
201 mydict = getconfig("bt_daemon.conf")
202 if not mydict:
203 print "bt_daemon.conf is missing"
204
205 if mydict.has_key("file_base") and mydict["file_base"]:
206 file_base = os.path.normpath(mydict["file_base"])
207 else:
208 print "file_base not specified in config."
209 sys.exit(1)
210
211 if mydict.has_key("torrent_base") and mydict["torrent_base"]:
212 torrent_base = os.path.normpath(mydict["torrent_base"])
213 else:
214 print "torrent_base not specified in config."
215 sys.exit(1)
216
217 threads = {}
218 dirs = [torrent_base[:],file_base[:]]
219 re_file = re.compile("("+torrent_base[:]+"|"+file_base[:]+")(/.+?)(\.torrent)?$")
220 absmax_upRate = 0.0
221 absmax_upFile = ''
222
223 loop_val = 0
224 while(True):
225 mydict = getconfig("bt_daemon.conf")
226 if not mydict:
227 sys.stderr.write("bt_daemon.conf is missing or contains errors\n")
228
229 if mydict.has_key("master_seed"):
230 master_seed = int(mydict["master_seed"])
231
232 if mydict.has_key("piece_size_pow2"):
233 piece_size_pow2 = int(mydict["piece_size_pow2"])
234 if mydict.has_key("max_starting_threads"):
235 max_starting_threads = int(mydict["max_starting_threads"])
236 if mydict.has_key("loop_mod"):
237 loop_mod = int(mydict["loop_mod"])
238 if mydict.has_key("rotate_logs"):
239 rotate_logs = int(mydict["rotate_logs"])
240 if mydict.has_key("cycle_sleep"):
241 cycle_sleep = int(mydict["cycle_sleep"])
242 if mydict.has_key("spawn_thread_delay"):
243 spawn_thread_delay = int(mydict["spawn_thread_delay"])
244 if mydict.has_key("debug"):
245 debug = int(mydict["debug"])
246
247 if mydict.has_key("min_uploads"):
248 min_uploads = int(mydict["min_uploads"])
249 if mydict.has_key("max_uploads"):
250 max_uploads = int(mydict["max_uploads"])
251 if mydict.has_key("download_slice_size"):
252 download_slice_size = int(mydict["download_slice_size"])
253 if mydict.has_key("max_slice_length"):
254 max_slice_length = int(mydict["max_slice_length"])
255 if mydict.has_key("request_backlog"):
256 request_backlog = int(mydict["request_backlog"])
257 if mydict.has_key("timeout"):
258 timeout = int(mydict["timeout"])
259 if mydict.has_key("timeout_check_interval"):
260 timeout_check_interval = int(mydict["timeout_check_interval"])
261
262 if mydict.has_key("comment"):
263 comment = mydict["comment"]
264 if mydict.has_key("port_min"):
265 port_min = mydict["port_min"]
266 if mydict.has_key("port_max"):
267 port_max = mydict["port_max"]
268 if mydict.has_key("check_hashes"):
269 check_hashes = mydict["check_hashes"]
270
271 if mydict.has_key("tracker") and mydict["tracker"]:
272 tracker = mydict["tracker"]
273 else:
274 print "tracker not specified in config."
275 sys.exit(1)
276
277
278 loop_val = (loop_val + 1) % loop_mod
279 files = list_files(dirs)
280 files.sort()
281 new_files = []
282 torrents = []
283
284 if shuffle_files:
285 random.shuffle(files)
286
287 for f in files:
288 try:
289 match = re_file.match(f)
290 if not match:
291 print "Failed to match:",f
292 else:
293 if match.group(1) == torrent_base:
294 if match.group(3): # It's a torrent, yay!
295 if match.group(2) not in torrents:
296 mystat = stat(f)
297 # only add if it has a size and is not actively modified.
298 if mystat[ST_SIZE]>0 and (time()-mystat[ST_MTIME])>minimum_age:
299 torrents.append(match.group(2))
300 while(match.group(2) in new_files):
301 del new_files[new_files.index(match.group(2))]
302 elif torrent_base != file_base and debug:
303 print "Not a torrent file:",f
304 if match.group(1) == file_base:
305 if match.group(3) and torrent_base != file_base and debug:
306 print "Torrent in file directory:",f
307 elif match.group(2) not in torrents:
308 mystat = stat(f)
309 if mystat[ST_SIZE]>0 and (time()-mystat[ST_MTIME])>minimum_age:
310 new_files.append(match.group(2))
311 if debug:
312 print "New file:",f
313 else:
314 if debug:
315 print "Actively modified: ",f
316 except Exception,e:
317 traceback.print_last()
318 print "--- IGNORING FILE:",t
319 print
320
321 for t in threads.keys():
322 # Remove threads that we don't have a bittorrent for.
323 if t not in torrents or not os.path.exists(file_base+t):
324 print "Stopping torrent [%s]: %s" % (ctime(),t)
325 threads[t][0]._Thread__stop()
326 del threads[t]
327 while(t in torrents):
328 del torrents[torrents.index(t)]
329
330 elif not threads[t][0].isAlive():
331 print "Thread died [%s]: %s" % (ctime(),t)
332 del threads[t]
333
334 completed = []
335 fetching = []
336 hashing = []
337 stalled = []
338
339 stats_up = []
340 stats_down = []
341
342 totalup = 0
343 rateup = 0
344 ratedown = 0
345
346 max_up_rate = 0
347 max_up_file = ''
348 max_down_rate = 0
349 max_down_file = ''
350 completed_string = ""
351 for t in threads.keys():
352 # Let's do some stats.
353 if not threads[t][1].failed and threads[t][0].isAlive():
354 if threads[t][1].done:
355 completed.append(t)
356 else:
357 fetching.append(t)
358 if not threads[t][1].active:
359 hashing.append(t)
360 elif threads[t][1].downRate == 0:
361 stalled.append(t)
362
363 stats_up.append("%s\tT:%f\tR:%03.2f\t%s" % (t,threads[t][1].upTotal,threads[t][1].upRate,threads[t][1].activity))
364 stats_down.append("%s\tT:%f\tR:%03.2f\t%s" % (t,threads[t][1].percentDone,threads[t][1].downRate,threads[t][1].activity))
365
366 totalup += threads[t][1].upTotal
367
368 rateup += threads[t][1].upRate
369 if threads[t][1].upRate > max_up_rate:
370 max_up_rate = threads[t][1].upRate
371 max_up_file = threads[t][1].file[:]
372 if threads[t][1].upRate > absmax_upRate:
373 absmax_upRate = threads[t][1].upRate
374 absmax_upFile = threads[t][1].file[:]
375
376 ratedown += threads[t][1].downRate
377 if threads[t][1].downRate > max_down_rate:
378 max_down_rate = threads[t][1].downRate
379 max_down_file = threads[t][1].file[:]
380
381 if stats:
382 if rotate_logs:
383 filename = "bt_daemon-%d-%04d.stats" % (os.getpid(),loop_val)
384 makenew = 1
385 else:
386 filename = "bt_daemon-%d.stats" % (os.getpid())
387 makenew = 0
388 writemsg("",filename,overwrite=makenew)
389 writemsg("Stats: %s" % (ctime()) ,filename)
390 writemsg("Seeding: %d" % (len(completed)) ,filename)
391 writemsg("Fetching: %d (stalled: %d) (hashing: %d)" % (len(fetching),len(stalled),len(hashing)),filename)
392 writemsg("Total up: %.3f MB" % (totalup) ,filename)
393 writemsg("UpRate: %.2f k/s" % (rateup) ,filename)
394 writemsg("UpMaxNow: %.2f k/s '%s'" % (max_up_rate,max_up_file) ,filename)
395 writemsg("UpAbsMax: %.2f k/s '%s'" % (absmax_upRate,absmax_upFile) ,filename)
396 writemsg("DownRate: %.2f k/s" % (ratedown) ,filename)
397 writemsg("DownBest: %.2f k/s '%s'" % (max_down_rate,max_down_file) ,filename)
398 writemsg("",filename)
399
400 completed.sort()
401 writemsg(string.join(completed, "\n"), filename+".completed", overwrite=1, echo=0)
402 hashing.sort()
403 writemsg(string.join(hashing, "\n"), filename+".hashing", overwrite=1, echo=0)
404 stalled.sort()
405 writemsg(string.join(stalled, "\n"), filename+".stalled", overwrite=1, echo=0)
406 stats_up.sort()
407 writemsg(string.join(stats_up, "\n"), filename+".stats_up", overwrite=1, echo=0)
408 stats_down.sort()
409 writemsg(string.join(stats_down, "\n"), filename+".stats_down", overwrite=1, echo=0)
410
411 for t in torrents:
412 # Start threads up if we have load free for them. Startup is harsh.
413 if t not in threads.keys() and (max_starting_threads - len(hashing) > 0):
414 hashing.append(t)
415 print "Starting torrent [%s]: %s" % (ctime(),t)
416 h = HeadlessDisplayer()
417 try:
418 if not os.path.isdir(file_base+"/"+os.path.dirname(t)):
419 os.makedirs(file_base+"/"+os.path.dirname(t))
420 download_args = [
421 # responsefile -> Torrent filename
422 '--responsefile', torrent_base+"/"+t+".torrent",
423 # saveas -> Real filename
424 '--saveas', file_base+"/"+t,
425 # max_slice_length -> How much we can send in a single request
426 '--max_slice_length', str(max_slice_length),
427 # timeout_check_interval -> Time between timeout checks
428 '--timeout_check_interval', str(timeout_check_interval),
429 # timeout -> How long until we time out a connection
430 '--timeout', str(timeout),
431 # minport -> Lowest port allowed for files
432 '--minport', str(port_min),
433 # maxport -> Highest port allowed for files
434 '--maxport', str(port_max),
435 # request_backlog -> How many requests per pipe do we hold
436 '--request_backlog', str(request_backlog),
437 # download_slice_size -> How much do we request?
438 '--download_slice_size', str(download_slice_size),
439 # max_uploads -> How many do we serve out?
440 '--max_uploads', str(max_uploads),
441 # min_uploads -> When do we switch from random to rare?
442 '--min_uploads', str(min_uploads),
443 # check_hashes -> Do we check the on-disk files before joining?
444 '--check_hashes', str(check_hashes)
445 ]
446 t_thread = Thread(target=download,name="BitTorrent::"+t,args=(download_args, h.chooseFile, h.display, h.finished, h.error, Event(), cols))
447 threads[t] = [t_thread,h]
448 t_thread.start()
449 sleep(spawn_thread_delay)
450 except Exception, e:
451 print "Failed to start thread: %s" % (t)
452 print "Reason: %s" % (str(e))
453
454 if master_seed:
455 for f in new_files:
456 # Create the torrents for files that don't have them.
457 try:
458 print "Creating torrent [%s]: %s" % (ctime(),f)
459 if not os.path.isdir(torrent_base+"/"+os.path.dirname(f)):
460 os.makedirs(torrent_base+"/"+os.path.dirname(f))
461 make_meta_file(file_base+"/"+f,tracker,piece_size_pow2,progress=progress,comment=comment,target=torrent_base+"/"+f+".torrent")
462 except Exception, e:
463 print "Failed to create torrent: "+str(e)
464
465 sleep(cycle_sleep)
466 except Exception, e:
467 import signal,traceback
468 print e
469 traceback.print_stack()
470 os.kill(0,signal.SIGTERM)

  ViewVC Help
Powered by ViewVC 1.1.20