/[gentoo-src]/bittorrent/bt_daemon.py
Gentoo

Contents of /bittorrent/bt_daemon.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.6 - (show annotations) (download) (as text)
Wed Jun 16 08:46:48 2004 UTC (10 years, 3 months ago) by carpaski
Branch: MAIN
CVS Tags: HEAD
Changes since 1.5: +68 -34 lines
File MIME type: text/x-python
Blah.

1 #!/usr/bin/python -O
2 # $Id: bt_daemon.py,v 1.5 2004/06/16 05:15:31 carpaski Exp $
3 #
4 # Greatly modified by carpaski@gentoo.org
5 # btdownloadheadless written by Bram Cohen
6 # see LICENSE.txt for license information
7
8 import os,sys,string,signal
9 os.nice(5)
10
11 sys.path = ["/usr/lib/portage/pym"] + sys.path + ["/usr/bin"]
12 from portage import getconfig
13
14 import BitTorrent
15 from BitTorrent.download import download
16
17 from threading import Event,Thread
18 import types,re,random
19
20 from os import listdir, getcwd, stat
21 from stat import ST_SIZE,ST_MTIME
22 from os.path import abspath,isdir,isfile,normpath
23 from sys import argv, stdout, path
24 from cStringIO import StringIO
25 from time import time,ctime,sleep
26
27 from btmakemetafile import make_meta_file
28
29 def hours(n):
30 if n == -1:
31 return '<unknown>'
32 if n == 0:
33 return 'complete!'
34 n = long(n)
35 h, r = divmod(n, 60 * 60)
36 m, sec = divmod(r, 60)
37 if h > 1000000:
38 return '<unknown>'
39 if h > 0:
40 return '%d hour %02d min %02d sec' % (h, m, sec)
41 else:
42 return '%d min %02d sec' % (m, sec)
43
44 class HeadlessDisplayer:
45 def __init__(self):
46 self.active = False
47 self.done = False
48 self.failed = False
49 self.file = ''
50 self.activity = ''
51 self.percentDone = 0.0
52 self.timeEst = 0
53 self.downloadTo = ''
54 self.downRate = 0.0
55 self.upRate = 0.0
56 self.downTotal = 0.0
57 self.upTotal = 0
58 self.errors = []
59 self.last_update_time = 0
60
61 def finished(self):
62 self.done = True
63 self.percentDone = 100
64 self.timeEst = 0
65 self.downRate = 0
66 self.display({})
67
68 def failed(self):
69 self.done = True
70 self.failed = True
71 self.percentDone = 0
72 self.timeEst = -1
73 self.downRate = 0
74 self.display({})
75
76 def error(self, errormsg):
77 self.errors.append(errormsg)
78 self.display({})
79
80 def display(self, dict):
81 if ((self.last_update_time + 1) > time()):
82 if dict.get('fractionDone') not in (0.0, 1.0):
83 if not dict.has_key('activity'):
84 return
85 self.last_update_time = time()
86 if dict.has_key('fractionDone'):
87 self.percentDone = float(int(dict['fractionDone'] * 1000)) / 10
88 if dict.has_key('timeEst'):
89 self.timeEst = long(dict['timeEst'])
90 if dict.has_key('activity') and not self.done:
91 self.activity = dict['activity']
92 if dict.has_key('downRate'):
93 self.downRate = float(dict['downRate']) / (1 << 10)
94 if self.downRate > 0:
95 self.has_transfered = True
96 if dict.has_key('upRate'):
97 self.upRate = float(dict['upRate']) / (1 << 10)
98 if self.upRate > 0:
99 self.has_transfered = True
100 if dict.has_key('upTotal'):
101 self.upTotal = dict['upTotal']
102 if dict.has_key('downTotal'):
103 self.active = True
104 self.downTotal = dict['downTotal']
105 for x in self.errors:
106 print self.file+": "+str(x)
107 self.errors = []
108 return
109
110 def chooseFile(self, default, size, saveas, dir):
111 self.file = '%s (%.1f MB)' % (default, float(size) / (1 << 20))
112 if saveas != '':
113 default = saveas
114 self.downloadTo = abspath(default)
115 return default
116
117 def newpath(self, path):
118 self.downloadTo = path
119
120 def prefix_array(array,prefix,doblanks=1):
121 """Prepends a given prefix to each element in an Array/List/Tuple.
122 Returns a List."""
123 if type(array) not in [types.ListType, types.TupleType]:
124 raise TypeError, "List or Tuple expected. Got %s" % type(array)
125 newarray=[]
126 for x in array:
127 if x or doblanks:
128 newarray.append(prefix + x)
129 else:
130 newarray.append(x)
131 return newarray
132
133 def list_files(dirlist):
134 file_list = []
135 for mydir in dirlist:
136 mys = mydir.split("/")
137 if mys[-1][0] == '.':
138 continue
139 if isdir(mydir):
140 try:
141 file_list += list_files(prefix_array(listdir(mydir),mydir+"/"))
142 except Exception,e:
143 print e
144 elif isfile(mydir):
145 file_list.append(mydir)
146 else:
147 print "Unknown type:",mydir
148 return file_list
149
150
151 def progress(amount):
152 # Dummy function for the create-torrent progress meter
153 pass
154
155
156 def writemsg(msg,fileName,overwrite=0,create=0,echo=1):
157 if overwrite or create:
158 fileHandle = open(fileName, "w")
159 else:
160 fileHandle = open(fileName, "a")
161
162 fileHandle.write(msg+"\n")
163 fileHandle.flush()
164 fileHandle.close()
165 if echo:
166 print msg
167
168
169 # Useless variables
170 cols = 80
171
172 # This is 2^x ... put the x below -- Chunk size for sha1 -- 18 == 256k
173 piece_size_pow2 = 18
174 max_starting_threads = 1
175 loop_mod = 4320 # Rotating value (for logs)
176 rotate_logs = 0 # Make one log or roate into loop_mod logs
177 cycle_sleep = 20
178 spawn_thread_delay = 0
179 debug = 0
180 minimum_age = 120
181 port_min = 6881
182 port_max = 7280
183 check_hashes = 0
184 shuffle_files = 0
185 master_seed = 0
186 min_uploads = 20
187 max_uploads = 20
188 download_slice_size = 65536
189 max_slice_length = 131072
190 request_backlog = 20
191 timeout = 90
192 timeout_check_interval = 20
193 stats = 1
194 min_file_size = 5000000
195 max_fetches = 3
196
197 # No trailing /
198 file_base = "/usr/portage/distfiles"
199 torrent_base = "/tmp/torrents"
200
201 comment = "Gentoo Linux BitTorrent Mirror System"
202 tracker = "http://egret.gentoo.org:6969/announce"
203
204 if __name__ == '__main__':
205 try:
206 mydict = getconfig("bt_daemon.conf")
207 if not mydict:
208 print "bt_daemon.conf is missing"
209
210 if mydict.has_key("file_base") and mydict["file_base"]:
211 file_base = os.path.normpath(mydict["file_base"])
212 else:
213 print "file_base not specified in config."
214 sys.exit(1)
215
216 if mydict.has_key("torrent_base") and mydict["torrent_base"]:
217 torrent_base = os.path.normpath(mydict["torrent_base"])
218 else:
219 print "torrent_base not specified in config."
220 sys.exit(1)
221
222 threads = {}
223 dirs = [torrent_base[:],file_base[:]]
224 re_file = re.compile("("+torrent_base[:]+"|"+file_base[:]+")(/.+?)(\.torrent)?$")
225 absmax_upRate = 0.0
226 absmax_upFile = ''
227
228 noticed_newfiles = []
229
230 loop_val = 0
231 while(True):
232 mydict = getconfig("bt_daemon.conf")
233 if not mydict:
234 sys.stderr.write("bt_daemon.conf is missing or contains errors\n")
235
236 if mydict.has_key("master_seed"):
237 master_seed = int(mydict["master_seed"])
238
239 if mydict.has_key("piece_size_pow2"):
240 piece_size_pow2 = int(mydict["piece_size_pow2"])
241 if mydict.has_key("max_starting_threads"):
242 max_starting_threads = int(mydict["max_starting_threads"])
243 if mydict.has_key("max_fetches"):
244 max_fetches = int(mydict["max_fetches"])
245 if mydict.has_key("loop_mod"):
246 loop_mod = int(mydict["loop_mod"])
247 if mydict.has_key("rotate_logs"):
248 rotate_logs = int(mydict["rotate_logs"])
249 if mydict.has_key("cycle_sleep"):
250 cycle_sleep = int(mydict["cycle_sleep"])
251 if mydict.has_key("spawn_thread_delay"):
252 spawn_thread_delay = int(mydict["spawn_thread_delay"])
253 if mydict.has_key("shuffle_files"):
254 shuffle_files = int(mydict["shuffle_files"])
255 if mydict.has_key("debug"):
256 debug = int(mydict["debug"])
257
258 if mydict.has_key("min_uploads"):
259 min_uploads = int(mydict["min_uploads"])
260 if mydict.has_key("max_uploads"):
261 max_uploads = int(mydict["max_uploads"])
262 if mydict.has_key("download_slice_size"):
263 download_slice_size = int(mydict["download_slice_size"])
264 if mydict.has_key("max_slice_length"):
265 max_slice_length = int(mydict["max_slice_length"])
266 if mydict.has_key("request_backlog"):
267 request_backlog = int(mydict["request_backlog"])
268 if mydict.has_key("timeout"):
269 timeout = int(mydict["timeout"])
270 if mydict.has_key("timeout_check_interval"):
271 timeout_check_interval = int(mydict["timeout_check_interval"])
272
273 if mydict.has_key("comment"):
274 comment = mydict["comment"]
275 if mydict.has_key("port_min"):
276 port_min = mydict["port_min"]
277 if mydict.has_key("port_max"):
278 port_max = mydict["port_max"]
279 if mydict.has_key("check_hashes"):
280 check_hashes = mydict["check_hashes"]
281
282 if mydict.has_key("tracker") and mydict["tracker"]:
283 tracker = mydict["tracker"]
284 else:
285 print "tracker not specified in config."
286 sys.exit(1)
287
288
289 loop_val = (loop_val + 1) % loop_mod
290 files = list_files(dirs)
291 files.sort()
292 new_files = []
293 torrents = []
294 changing = []
295
296 if shuffle_files:
297 random.shuffle(files)
298
299 for f in files:
300 try:
301 match = re_file.match(f)
302 if not match:
303 print "Failed to match:",f
304 else:
305 if match.group(1) == torrent_base:
306 if match.group(3): # It's a torrent, yay!
307 if match.group(2) not in torrents:
308 mystat = stat(f)
309 # only add if it has a size and is not actively modified.
310 if mystat[ST_SIZE]>0 and (time()-mystat[ST_MTIME])>minimum_age:
311 torrents.append(match.group(2))
312 else:
313 changing.append(match.group(2))
314 while(match.group(2) in new_files):
315 del new_files[new_files.index(match.group(2))]
316 elif torrent_base != file_base and debug:
317 print "Not a torrent file:",f
318 if match.group(1) == file_base:
319 if match.group(3) and torrent_base != file_base and debug:
320 print "Torrent in file directory:",f
321 elif match.group(2) not in torrents:
322 mystat = stat(f)
323 if mystat[ST_SIZE] < min_file_size:
324 pass # Too small ignore it.
325 elif (time()-mystat[ST_MTIME]) > minimum_age:
326 new_files.append(match.group(2))
327 if debug:
328 if new_files[-1] not in torrents:
329 if new_files[-1] not in changing:
330 if f not in noticed_newfiles:
331 print "New file:",f
332 noticed_newfiles.append(f)
333 else:
334 print "Changing file:",f
335 else:
336 if debug:
337 print "Actively modified: ",f
338 except Exception,e:
339 print "--- IGNORING FILE: %s '%s'" % (f,e)
340 print
341
342 for t in threads.keys():
343 # Remove threads that we don't have a bittorrent for.
344 if t not in torrents or not os.path.exists(file_base+t):
345 print "Stopping torrent [%s]: %s" % (ctime(),t)
346 threads[t][0]._Thread__stop()
347 del threads[t]
348 while(t in torrents):
349 del torrents[torrents.index(t)]
350
351 elif not threads[t][0].isAlive():
352 print "Thread died [%s]: %s" % (ctime(),t)
353 del threads[t]
354
355 completed = []
356 fetching = []
357 hashing = []
358 stalled = []
359
360 stats_up = []
361 stats_down = []
362
363 totalup = 0
364 rateup = 0
365 ratedown = 0
366
367 max_up_rate = 0
368 max_up_file = ''
369 max_down_rate = 0
370 max_down_file = ''
371 completed_string = ""
372 for t in threads.keys():
373 # Let's do some stats.
374 if not threads[t][1].failed and threads[t][0].isAlive():
375 if threads[t][1].done:
376 completed.append(t)
377 else:
378 fetching.append(t)
379 if not threads[t][1].active:
380 hashing.append(t)
381 elif threads[t][1].downRate == 0:
382 stalled.append(t)
383
384 stats_up.append("%s\tT:%f\tR:%03.2f\t%s" % (t,threads[t][1].upTotal,threads[t][1].upRate,threads[t][1].activity))
385 stats_down.append("%s\tT:%f\tR:%03.2f\t%s" % (t,threads[t][1].percentDone,threads[t][1].downRate,threads[t][1].activity))
386
387 totalup += threads[t][1].upTotal
388
389 rateup += threads[t][1].upRate
390 if threads[t][1].upRate > max_up_rate:
391 max_up_rate = threads[t][1].upRate
392 max_up_file = threads[t][1].file[:]
393 if threads[t][1].upRate > absmax_upRate:
394 absmax_upRate = threads[t][1].upRate
395 absmax_upFile = threads[t][1].file[:]
396
397 ratedown += threads[t][1].downRate
398 if threads[t][1].downRate > max_down_rate:
399 max_down_rate = threads[t][1].downRate
400 max_down_file = threads[t][1].file[:]
401
402 if stats:
403 if rotate_logs:
404 filename = "bt_daemon-%d-%04d.stats" % (os.getpid(),loop_val)
405 makenew = 1
406 else:
407 filename = "bt_daemon-%d.stats" % (os.getpid())
408 makenew = 0
409 writemsg("",filename,overwrite=makenew)
410 writemsg("Stats: %s" % (ctime()) ,filename)
411 writemsg("Seeding: %d" % (len(completed)) ,filename)
412 writemsg("Fetching: %d/%d (stalled: %d) (hashing: %d)" % (len(fetching),max_fetches,len(stalled),len(hashing)),filename)
413 writemsg("Total up: %.3f MB" % (totalup) ,filename)
414 writemsg("UpRate: %.2f k/s" % (rateup) ,filename)
415 writemsg("UpMaxNow: %.2f k/s '%s'" % (max_up_rate,max_up_file) ,filename)
416 writemsg("UpAbsMax: %.2f k/s '%s'" % (absmax_upRate,absmax_upFile) ,filename)
417 writemsg("DownRate: %.2f k/s" % (ratedown) ,filename)
418 writemsg("DownBest: %.2f k/s '%s'" % (max_down_rate,max_down_file) ,filename)
419 writemsg("",filename)
420
421 completed.sort()
422 writemsg(string.join(completed, "\n"), filename+".completed", overwrite=1, echo=0)
423 hashing.sort()
424 writemsg(string.join(hashing, "\n"), filename+".hashing", overwrite=1, echo=0)
425 stalled.sort()
426 writemsg(string.join(stalled, "\n"), filename+".stalled", overwrite=1, echo=0)
427 stats_up.sort()
428 writemsg(string.join(stats_up, "\n"), filename+".stats_up", overwrite=1, echo=0)
429 stats_down.sort()
430 writemsg(string.join(stats_down, "\n"), filename+".stats_down", overwrite=1, echo=0)
431
432 for t in torrents:
433 # Start threads up if we have load free for them. Startup is harsh.
434 if len(fetching) >= max_fetches:
435 continue
436 if t not in threads.keys() and (max_starting_threads - len(hashing) > 0):
437 hashing.append(t)
438 while t in noticed_newfiles:
439 del noticed_newfiles[noticed_newfiles.index(t)]
440 print "Starting torrent [%s]: %s" % (ctime(),t)
441 h = HeadlessDisplayer()
442 try:
443 if not os.path.isdir(file_base+"/"+os.path.dirname(t)):
444 os.makedirs(file_base+"/"+os.path.dirname(t))
445 download_args = [
446 # responsefile -> Torrent filename
447 '--responsefile', torrent_base+"/"+t+".torrent",
448 # saveas -> Real filename
449 '--saveas', file_base+"/"+t,
450 # max_slice_length -> How much we can send in a single request
451 '--max_slice_length', str(max_slice_length),
452 # timeout_check_interval -> Time between timeout checks
453 '--timeout_check_interval', str(timeout_check_interval),
454 # timeout -> How long until we time out a connection
455 '--timeout', str(timeout),
456 # minport -> Lowest port allowed for files
457 '--minport', str(port_min),
458 # maxport -> Highest port allowed for files
459 '--maxport', str(port_max),
460 # request_backlog -> How many requests per pipe do we hold
461 '--request_backlog', str(request_backlog),
462 # download_slice_size -> How much do we request?
463 '--download_slice_size', str(download_slice_size),
464 # max_uploads -> How many do we serve out?
465 '--max_uploads', str(max_uploads),
466 # min_uploads -> When do we switch from random to rare?
467 '--min_uploads', str(min_uploads),
468 # check_hashes -> Do we check the on-disk files before joining?
469 '--check_hashes', str(check_hashes)
470 ]
471 t_thread = Thread(target=download,name="BitTorrent::"+t,args=(download_args, h.chooseFile, h.display, h.finished, h.error, Event(), cols))
472 threads[t] = [t_thread,h]
473 t_thread.start()
474 sleep(spawn_thread_delay)
475 except Exception, e:
476 print "Failed to start thread: %s" % (t)
477 print "Reason: %s" % (str(e))
478
479 if master_seed:
480 for f in new_files:
481 if f in torrents:
482 print "in torrents:",f
483 continue
484 # Create the torrents for files that don't have them.
485 mystat = os.stat(file_base+"/"+f)
486 if mystat[ST_SIZE] < min_file_size:
487 print "Too small:",f
488 continue
489 # only add if it has a size and is not actively modified.
490 if (time() - mystat[ST_MTIME]) < minimum_age:
491 print "Too new:",f
492 continue
493 try:
494 print "Creating torrent [%s]: %s" % (ctime(),f)
495 if not os.path.isdir(torrent_base+"/"+os.path.dirname(f)):
496 os.makedirs(torrent_base+"/"+os.path.dirname(f))
497 make_meta_file(file_base+"/"+f,tracker,piece_size_pow2,progress=progress,comment=comment,target=torrent_base+"/"+f+".torrent")
498 except Exception, e:
499 print "Failed to create torrent: "+str(e)
500
501 sleep(cycle_sleep)
502 except Exception, e:
503 sys.stderr.write("\n\nException caught. Terminating...\n%s\n\n" % e)
504 os.kill(0,signal.SIGTERM)

  ViewVC Help
Powered by ViewVC 1.1.20