/[gentoo-src]/bittorrent/bt_daemon.py
Gentoo

Contents of /bittorrent/bt_daemon.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.6 - (hide annotations) (download) (as text)
Wed Jun 16 08:46:48 2004 UTC (9 years, 10 months ago) by carpaski
Branch: MAIN
CVS Tags: HEAD
Changes since 1.5: +68 -34 lines
File MIME type: text/x-python
Blah.

1 carpaski 1.1 #!/usr/bin/python -O
2 carpaski 1.6 # $Id: bt_daemon.py,v 1.5 2004/06/16 05:15:31 carpaski Exp $
3 carpaski 1.4 #
4 carpaski 1.1 # Greatly modified by carpaski@gentoo.org
5     # btdownloadheadless written by Bram Cohen
6     # see LICENSE.txt for license information
7    
8 carpaski 1.6 import os,sys,string,signal
9 carpaski 1.2 os.nice(5)
10 carpaski 1.1
11 carpaski 1.4 sys.path = ["/usr/lib/portage/pym"] + sys.path + ["/usr/bin"]
12     from portage import getconfig
13 carpaski 1.2
14 carpaski 1.1 import BitTorrent
15     from BitTorrent.download import download
16 carpaski 1.5
17 carpaski 1.1 from threading import Event,Thread
18 carpaski 1.6 import types,re,random
19 carpaski 1.5
20 carpaski 1.1 from os import listdir, getcwd, stat
21 carpaski 1.4 from stat import ST_SIZE,ST_MTIME
22 carpaski 1.1 from os.path import abspath,isdir,isfile,normpath
23     from sys import argv, stdout, path
24     from cStringIO import StringIO
25     from time import time,ctime,sleep
26    
27 carpaski 1.4 from btmakemetafile import make_meta_file
28 carpaski 1.1
29     def hours(n):
30 carpaski 1.2 if n == -1:
31     return '<unknown>'
32     if n == 0:
33     return 'complete!'
34     n = long(n)
35     h, r = divmod(n, 60 * 60)
36     m, sec = divmod(r, 60)
37     if h > 1000000:
38     return '<unknown>'
39     if h > 0:
40     return '%d hour %02d min %02d sec' % (h, m, sec)
41     else:
42     return '%d min %02d sec' % (m, sec)
43 carpaski 1.1
44     class HeadlessDisplayer:
45 carpaski 1.2 def __init__(self):
46 carpaski 1.5 self.active = False
47 carpaski 1.2 self.done = False
48     self.failed = False
49     self.file = ''
50     self.activity = ''
51     self.percentDone = 0.0
52     self.timeEst = 0
53     self.downloadTo = ''
54     self.downRate = 0.0
55     self.upRate = 0.0
56     self.downTotal = 0.0
57     self.upTotal = 0
58     self.errors = []
59     self.last_update_time = 0
60    
61     def finished(self):
62     self.done = True
63     self.percentDone = 100
64     self.timeEst = 0
65     self.downRate = 0
66     self.display({})
67    
68     def failed(self):
69     self.done = True
70     self.failed = True
71     self.percentDone = 0
72     self.timeEst = -1
73     self.downRate = 0
74     self.display({})
75    
76     def error(self, errormsg):
77     self.errors.append(errormsg)
78     self.display({})
79    
80     def display(self, dict):
81     if ((self.last_update_time + 1) > time()):
82     if dict.get('fractionDone') not in (0.0, 1.0):
83     if not dict.has_key('activity'):
84     return
85     self.last_update_time = time()
86     if dict.has_key('fractionDone'):
87     self.percentDone = float(int(dict['fractionDone'] * 1000)) / 10
88     if dict.has_key('timeEst'):
89     self.timeEst = long(dict['timeEst'])
90     if dict.has_key('activity') and not self.done:
91     self.activity = dict['activity']
92     if dict.has_key('downRate'):
93     self.downRate = float(dict['downRate']) / (1 << 10)
94 carpaski 1.5 if self.downRate > 0:
95     self.has_transfered = True
96 carpaski 1.2 if dict.has_key('upRate'):
97     self.upRate = float(dict['upRate']) / (1 << 10)
98 carpaski 1.5 if self.upRate > 0:
99     self.has_transfered = True
100 carpaski 1.2 if dict.has_key('upTotal'):
101     self.upTotal = dict['upTotal']
102     if dict.has_key('downTotal'):
103 carpaski 1.5 self.active = True
104 carpaski 1.2 self.downTotal = dict['downTotal']
105     for x in self.errors:
106     print self.file+": "+str(x)
107     self.errors = []
108     return
109    
110     def chooseFile(self, default, size, saveas, dir):
111     self.file = '%s (%.1f MB)' % (default, float(size) / (1 << 20))
112     if saveas != '':
113     default = saveas
114     self.downloadTo = abspath(default)
115     return default
116 carpaski 1.1
117 carpaski 1.2 def newpath(self, path):
118     self.downloadTo = path
119 carpaski 1.1
120     def prefix_array(array,prefix,doblanks=1):
121 carpaski 1.2 """Prepends a given prefix to each element in an Array/List/Tuple.
122     Returns a List."""
123     if type(array) not in [types.ListType, types.TupleType]:
124     raise TypeError, "List or Tuple expected. Got %s" % type(array)
125     newarray=[]
126     for x in array:
127     if x or doblanks:
128     newarray.append(prefix + x)
129     else:
130     newarray.append(x)
131     return newarray
132 carpaski 1.1
133     def list_files(dirlist):
134     file_list = []
135     for mydir in dirlist:
136 carpaski 1.6 mys = mydir.split("/")
137     if mys[-1][0] == '.':
138     continue
139 carpaski 1.1 if isdir(mydir):
140     try:
141     file_list += list_files(prefix_array(listdir(mydir),mydir+"/"))
142     except Exception,e:
143     print e
144     elif isfile(mydir):
145     file_list.append(mydir)
146     else:
147     print "Unknown type:",mydir
148     return file_list
149    
150    
151     def progress(amount):
152     # Dummy function for the create-torrent progress meter
153     pass
154    
155    
156 carpaski 1.5 def writemsg(msg,fileName,overwrite=0,create=0,echo=1):
157 carpaski 1.2 if overwrite or create:
158     fileHandle = open(fileName, "w")
159     else:
160     fileHandle = open(fileName, "a")
161    
162     fileHandle.write(msg+"\n")
163     fileHandle.flush()
164     fileHandle.close()
165 carpaski 1.5 if echo:
166     print msg
167 carpaski 1.1
168 carpaski 1.4
169     # Useless variables
170     cols = 80
171    
172     # This is 2^x ... put the x below -- Chunk size for sha1 -- 18 == 256k
173 carpaski 1.6 piece_size_pow2 = 18
174     max_starting_threads = 1
175     loop_mod = 4320 # Rotating value (for logs)
176     rotate_logs = 0 # Make one log or roate into loop_mod logs
177     cycle_sleep = 20
178     spawn_thread_delay = 0
179     debug = 0
180     minimum_age = 120
181     port_min = 6881
182     port_max = 7280
183     check_hashes = 0
184     shuffle_files = 0
185     master_seed = 0
186     min_uploads = 20
187     max_uploads = 20
188     download_slice_size = 65536
189     max_slice_length = 131072
190     request_backlog = 20
191     timeout = 90
192     timeout_check_interval = 20
193     stats = 1
194     min_file_size = 5000000
195     max_fetches = 3
196 carpaski 1.4
197     # No trailing /
198     file_base = "/usr/portage/distfiles"
199     torrent_base = "/tmp/torrents"
200    
201     comment = "Gentoo Linux BitTorrent Mirror System"
202     tracker = "http://egret.gentoo.org:6969/announce"
203    
204 carpaski 1.1 if __name__ == '__main__':
205 carpaski 1.5 try:
206 carpaski 1.4 mydict = getconfig("bt_daemon.conf")
207     if not mydict:
208     print "bt_daemon.conf is missing"
209 carpaski 1.5
210     if mydict.has_key("file_base") and mydict["file_base"]:
211     file_base = os.path.normpath(mydict["file_base"])
212     else:
213     print "file_base not specified in config."
214 carpaski 1.4 sys.exit(1)
215 carpaski 1.5
216     if mydict.has_key("torrent_base") and mydict["torrent_base"]:
217     torrent_base = os.path.normpath(mydict["torrent_base"])
218 carpaski 1.4 else:
219 carpaski 1.5 print "torrent_base not specified in config."
220 carpaski 1.4 sys.exit(1)
221 carpaski 1.5
222     threads = {}
223     dirs = [torrent_base[:],file_base[:]]
224     re_file = re.compile("("+torrent_base[:]+"|"+file_base[:]+")(/.+?)(\.torrent)?$")
225     absmax_upRate = 0.0
226     absmax_upFile = ''
227 carpaski 1.6
228     noticed_newfiles = []
229 carpaski 1.5
230     loop_val = 0
231     while(True):
232     mydict = getconfig("bt_daemon.conf")
233     if not mydict:
234     sys.stderr.write("bt_daemon.conf is missing or contains errors\n")
235    
236     if mydict.has_key("master_seed"):
237     master_seed = int(mydict["master_seed"])
238    
239     if mydict.has_key("piece_size_pow2"):
240     piece_size_pow2 = int(mydict["piece_size_pow2"])
241     if mydict.has_key("max_starting_threads"):
242     max_starting_threads = int(mydict["max_starting_threads"])
243 carpaski 1.6 if mydict.has_key("max_fetches"):
244     max_fetches = int(mydict["max_fetches"])
245 carpaski 1.5 if mydict.has_key("loop_mod"):
246     loop_mod = int(mydict["loop_mod"])
247     if mydict.has_key("rotate_logs"):
248     rotate_logs = int(mydict["rotate_logs"])
249     if mydict.has_key("cycle_sleep"):
250     cycle_sleep = int(mydict["cycle_sleep"])
251     if mydict.has_key("spawn_thread_delay"):
252     spawn_thread_delay = int(mydict["spawn_thread_delay"])
253 carpaski 1.6 if mydict.has_key("shuffle_files"):
254     shuffle_files = int(mydict["shuffle_files"])
255 carpaski 1.5 if mydict.has_key("debug"):
256     debug = int(mydict["debug"])
257    
258     if mydict.has_key("min_uploads"):
259     min_uploads = int(mydict["min_uploads"])
260     if mydict.has_key("max_uploads"):
261     max_uploads = int(mydict["max_uploads"])
262     if mydict.has_key("download_slice_size"):
263     download_slice_size = int(mydict["download_slice_size"])
264     if mydict.has_key("max_slice_length"):
265     max_slice_length = int(mydict["max_slice_length"])
266     if mydict.has_key("request_backlog"):
267     request_backlog = int(mydict["request_backlog"])
268     if mydict.has_key("timeout"):
269     timeout = int(mydict["timeout"])
270     if mydict.has_key("timeout_check_interval"):
271     timeout_check_interval = int(mydict["timeout_check_interval"])
272    
273     if mydict.has_key("comment"):
274     comment = mydict["comment"]
275     if mydict.has_key("port_min"):
276     port_min = mydict["port_min"]
277     if mydict.has_key("port_max"):
278     port_max = mydict["port_max"]
279     if mydict.has_key("check_hashes"):
280     check_hashes = mydict["check_hashes"]
281    
282     if mydict.has_key("tracker") and mydict["tracker"]:
283     tracker = mydict["tracker"]
284 carpaski 1.1 else:
285 carpaski 1.5 print "tracker not specified in config."
286     sys.exit(1)
287    
288    
289     loop_val = (loop_val + 1) % loop_mod
290     files = list_files(dirs)
291     files.sort()
292     new_files = []
293     torrents = []
294 carpaski 1.6 changing = []
295 carpaski 1.5
296     if shuffle_files:
297     random.shuffle(files)
298    
299     for f in files:
300     try:
301     match = re_file.match(f)
302     if not match:
303     print "Failed to match:",f
304     else:
305     if match.group(1) == torrent_base:
306     if match.group(3): # It's a torrent, yay!
307     if match.group(2) not in torrents:
308     mystat = stat(f)
309     # only add if it has a size and is not actively modified.
310     if mystat[ST_SIZE]>0 and (time()-mystat[ST_MTIME])>minimum_age:
311     torrents.append(match.group(2))
312 carpaski 1.6 else:
313     changing.append(match.group(2))
314 carpaski 1.5 while(match.group(2) in new_files):
315     del new_files[new_files.index(match.group(2))]
316     elif torrent_base != file_base and debug:
317     print "Not a torrent file:",f
318     if match.group(1) == file_base:
319     if match.group(3) and torrent_base != file_base and debug:
320     print "Torrent in file directory:",f
321     elif match.group(2) not in torrents:
322     mystat = stat(f)
323 carpaski 1.6 if mystat[ST_SIZE] < min_file_size:
324     pass # Too small ignore it.
325     elif (time()-mystat[ST_MTIME]) > minimum_age:
326     new_files.append(match.group(2))
327     if debug:
328     if new_files[-1] not in torrents:
329     if new_files[-1] not in changing:
330     if f not in noticed_newfiles:
331     print "New file:",f
332     noticed_newfiles.append(f)
333     else:
334     print "Changing file:",f
335 carpaski 1.5 else:
336     if debug:
337     print "Actively modified: ",f
338     except Exception,e:
339 carpaski 1.6 print "--- IGNORING FILE: %s '%s'" % (f,e)
340 carpaski 1.5 print
341    
342     for t in threads.keys():
343     # Remove threads that we don't have a bittorrent for.
344     if t not in torrents or not os.path.exists(file_base+t):
345     print "Stopping torrent [%s]: %s" % (ctime(),t)
346     threads[t][0]._Thread__stop()
347     del threads[t]
348     while(t in torrents):
349     del torrents[torrents.index(t)]
350    
351     elif not threads[t][0].isAlive():
352     print "Thread died [%s]: %s" % (ctime(),t)
353     del threads[t]
354    
355     completed = []
356     fetching = []
357     hashing = []
358     stalled = []
359    
360     stats_up = []
361     stats_down = []
362    
363     totalup = 0
364     rateup = 0
365     ratedown = 0
366    
367     max_up_rate = 0
368     max_up_file = ''
369     max_down_rate = 0
370     max_down_file = ''
371     completed_string = ""
372     for t in threads.keys():
373     # Let's do some stats.
374     if not threads[t][1].failed and threads[t][0].isAlive():
375     if threads[t][1].done:
376     completed.append(t)
377     else:
378     fetching.append(t)
379     if not threads[t][1].active:
380     hashing.append(t)
381     elif threads[t][1].downRate == 0:
382     stalled.append(t)
383    
384     stats_up.append("%s\tT:%f\tR:%03.2f\t%s" % (t,threads[t][1].upTotal,threads[t][1].upRate,threads[t][1].activity))
385     stats_down.append("%s\tT:%f\tR:%03.2f\t%s" % (t,threads[t][1].percentDone,threads[t][1].downRate,threads[t][1].activity))
386    
387     totalup += threads[t][1].upTotal
388    
389     rateup += threads[t][1].upRate
390     if threads[t][1].upRate > max_up_rate:
391     max_up_rate = threads[t][1].upRate
392     max_up_file = threads[t][1].file[:]
393     if threads[t][1].upRate > absmax_upRate:
394     absmax_upRate = threads[t][1].upRate
395     absmax_upFile = threads[t][1].file[:]
396    
397     ratedown += threads[t][1].downRate
398     if threads[t][1].downRate > max_down_rate:
399     max_down_rate = threads[t][1].downRate
400     max_down_file = threads[t][1].file[:]
401    
402     if stats:
403     if rotate_logs:
404     filename = "bt_daemon-%d-%04d.stats" % (os.getpid(),loop_val)
405     makenew = 1
406 carpaski 1.2 else:
407 carpaski 1.5 filename = "bt_daemon-%d.stats" % (os.getpid())
408     makenew = 0
409     writemsg("",filename,overwrite=makenew)
410     writemsg("Stats: %s" % (ctime()) ,filename)
411     writemsg("Seeding: %d" % (len(completed)) ,filename)
412 carpaski 1.6 writemsg("Fetching: %d/%d (stalled: %d) (hashing: %d)" % (len(fetching),max_fetches,len(stalled),len(hashing)),filename)
413 carpaski 1.5 writemsg("Total up: %.3f MB" % (totalup) ,filename)
414     writemsg("UpRate: %.2f k/s" % (rateup) ,filename)
415     writemsg("UpMaxNow: %.2f k/s '%s'" % (max_up_rate,max_up_file) ,filename)
416     writemsg("UpAbsMax: %.2f k/s '%s'" % (absmax_upRate,absmax_upFile) ,filename)
417     writemsg("DownRate: %.2f k/s" % (ratedown) ,filename)
418     writemsg("DownBest: %.2f k/s '%s'" % (max_down_rate,max_down_file) ,filename)
419     writemsg("",filename)
420 carpaski 1.2
421 carpaski 1.5 completed.sort()
422     writemsg(string.join(completed, "\n"), filename+".completed", overwrite=1, echo=0)
423     hashing.sort()
424     writemsg(string.join(hashing, "\n"), filename+".hashing", overwrite=1, echo=0)
425     stalled.sort()
426     writemsg(string.join(stalled, "\n"), filename+".stalled", overwrite=1, echo=0)
427     stats_up.sort()
428     writemsg(string.join(stats_up, "\n"), filename+".stats_up", overwrite=1, echo=0)
429     stats_down.sort()
430     writemsg(string.join(stats_down, "\n"), filename+".stats_down", overwrite=1, echo=0)
431    
432     for t in torrents:
433     # Start threads up if we have load free for them. Startup is harsh.
434 carpaski 1.6 if len(fetching) >= max_fetches:
435     continue
436 carpaski 1.5 if t not in threads.keys() and (max_starting_threads - len(hashing) > 0):
437     hashing.append(t)
438 carpaski 1.6 while t in noticed_newfiles:
439     del noticed_newfiles[noticed_newfiles.index(t)]
440 carpaski 1.5 print "Starting torrent [%s]: %s" % (ctime(),t)
441     h = HeadlessDisplayer()
442     try:
443     if not os.path.isdir(file_base+"/"+os.path.dirname(t)):
444     os.makedirs(file_base+"/"+os.path.dirname(t))
445     download_args = [
446     # responsefile -> Torrent filename
447     '--responsefile', torrent_base+"/"+t+".torrent",
448     # saveas -> Real filename
449     '--saveas', file_base+"/"+t,
450     # max_slice_length -> How much we can send in a single request
451     '--max_slice_length', str(max_slice_length),
452     # timeout_check_interval -> Time between timeout checks
453     '--timeout_check_interval', str(timeout_check_interval),
454     # timeout -> How long until we time out a connection
455     '--timeout', str(timeout),
456     # minport -> Lowest port allowed for files
457     '--minport', str(port_min),
458     # maxport -> Highest port allowed for files
459     '--maxport', str(port_max),
460     # request_backlog -> How many requests per pipe do we hold
461     '--request_backlog', str(request_backlog),
462     # download_slice_size -> How much do we request?
463     '--download_slice_size', str(download_slice_size),
464     # max_uploads -> How many do we serve out?
465     '--max_uploads', str(max_uploads),
466     # min_uploads -> When do we switch from random to rare?
467     '--min_uploads', str(min_uploads),
468     # check_hashes -> Do we check the on-disk files before joining?
469     '--check_hashes', str(check_hashes)
470     ]
471     t_thread = Thread(target=download,name="BitTorrent::"+t,args=(download_args, h.chooseFile, h.display, h.finished, h.error, Event(), cols))
472     threads[t] = [t_thread,h]
473     t_thread.start()
474     sleep(spawn_thread_delay)
475     except Exception, e:
476     print "Failed to start thread: %s" % (t)
477     print "Reason: %s" % (str(e))
478    
479     if master_seed:
480     for f in new_files:
481 carpaski 1.6 if f in torrents:
482     print "in torrents:",f
483     continue
484 carpaski 1.5 # Create the torrents for files that don't have them.
485 carpaski 1.6 mystat = os.stat(file_base+"/"+f)
486     if mystat[ST_SIZE] < min_file_size:
487     print "Too small:",f
488     continue
489     # only add if it has a size and is not actively modified.
490     if (time() - mystat[ST_MTIME]) < minimum_age:
491     print "Too new:",f
492     continue
493 carpaski 1.5 try:
494     print "Creating torrent [%s]: %s" % (ctime(),f)
495     if not os.path.isdir(torrent_base+"/"+os.path.dirname(f)):
496     os.makedirs(torrent_base+"/"+os.path.dirname(f))
497     make_meta_file(file_base+"/"+f,tracker,piece_size_pow2,progress=progress,comment=comment,target=torrent_base+"/"+f+".torrent")
498     except Exception, e:
499     print "Failed to create torrent: "+str(e)
500    
501     sleep(cycle_sleep)
502     except Exception, e:
503 carpaski 1.6 sys.stderr.write("\n\nException caught. Terminating...\n%s\n\n" % e)
504 carpaski 1.5 os.kill(0,signal.SIGTERM)

  ViewVC Help
Powered by ViewVC 1.1.20