/[gentoo-src]/bittorrent/bt_daemon.py
Gentoo

Contents of /bittorrent/bt_daemon.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.5 - (hide annotations) (download) (as text)
Wed Jun 16 05:15:31 2004 UTC (9 years, 10 months ago) by carpaski
Branch: MAIN
Changes since 1.4: +297 -220 lines
File MIME type: text/x-python
Updates to config file and descriptions.

1 carpaski 1.1 #!/usr/bin/python -O
2 carpaski 1.5 # $Id: bt_daemon.py,v 1.4 2004/05/25 23:51:44 carpaski Exp $
3 carpaski 1.4 #
4 carpaski 1.1 # Greatly modified by carpaski@gentoo.org
5     # btdownloadheadless written by Bram Cohen
6     # see LICENSE.txt for license information
7    
8 carpaski 1.5 import os,sys,string
9 carpaski 1.2 os.nice(5)
10 carpaski 1.1
11 carpaski 1.4 sys.path = ["/usr/lib/portage/pym"] + sys.path + ["/usr/bin"]
12     from portage import getconfig
13 carpaski 1.2
14 carpaski 1.1 import BitTorrent
15     from BitTorrent.download import download
16 carpaski 1.5
17 carpaski 1.1 from threading import Event,Thread
18 carpaski 1.5 import types,traceback,re,random
19    
20 carpaski 1.1 from os import listdir, getcwd, stat
21 carpaski 1.4 from stat import ST_SIZE,ST_MTIME
22 carpaski 1.1 from os.path import abspath,isdir,isfile,normpath
23     from sys import argv, stdout, path
24     from cStringIO import StringIO
25     from time import time,ctime,sleep
26    
27 carpaski 1.4 from btmakemetafile import make_meta_file
28 carpaski 1.1
29     def hours(n):
30 carpaski 1.2 if n == -1:
31     return '<unknown>'
32     if n == 0:
33     return 'complete!'
34     n = long(n)
35     h, r = divmod(n, 60 * 60)
36     m, sec = divmod(r, 60)
37     if h > 1000000:
38     return '<unknown>'
39     if h > 0:
40     return '%d hour %02d min %02d sec' % (h, m, sec)
41     else:
42     return '%d min %02d sec' % (m, sec)
43 carpaski 1.1
44     class HeadlessDisplayer:
45 carpaski 1.2 def __init__(self):
46 carpaski 1.5 self.active = False
47 carpaski 1.2 self.done = False
48     self.failed = False
49     self.file = ''
50     self.activity = ''
51     self.percentDone = 0.0
52     self.timeEst = 0
53     self.downloadTo = ''
54     self.downRate = 0.0
55     self.upRate = 0.0
56     self.downTotal = 0.0
57     self.upTotal = 0
58     self.errors = []
59     self.last_update_time = 0
60    
61     def finished(self):
62     self.done = True
63     self.percentDone = 100
64     self.timeEst = 0
65     self.downRate = 0
66     self.display({})
67    
68     def failed(self):
69     self.done = True
70     self.failed = True
71     self.percentDone = 0
72     self.timeEst = -1
73     self.downRate = 0
74     self.display({})
75    
76     def error(self, errormsg):
77     self.errors.append(errormsg)
78     self.display({})
79    
80     def display(self, dict):
81     if ((self.last_update_time + 1) > time()):
82     if dict.get('fractionDone') not in (0.0, 1.0):
83     if not dict.has_key('activity'):
84     return
85     self.last_update_time = time()
86     if dict.has_key('fractionDone'):
87     self.percentDone = float(int(dict['fractionDone'] * 1000)) / 10
88     if dict.has_key('timeEst'):
89     self.timeEst = long(dict['timeEst'])
90     if dict.has_key('activity') and not self.done:
91     self.activity = dict['activity']
92     if dict.has_key('downRate'):
93     self.downRate = float(dict['downRate']) / (1 << 10)
94 carpaski 1.5 if self.downRate > 0:
95     self.has_transfered = True
96 carpaski 1.2 if dict.has_key('upRate'):
97     self.upRate = float(dict['upRate']) / (1 << 10)
98 carpaski 1.5 if self.upRate > 0:
99     self.has_transfered = True
100 carpaski 1.2 if dict.has_key('upTotal'):
101     self.upTotal = dict['upTotal']
102     if dict.has_key('downTotal'):
103 carpaski 1.5 self.active = True
104 carpaski 1.2 self.downTotal = dict['downTotal']
105     for x in self.errors:
106     print self.file+": "+str(x)
107     self.errors = []
108     return
109    
110     def chooseFile(self, default, size, saveas, dir):
111     self.file = '%s (%.1f MB)' % (default, float(size) / (1 << 20))
112     if saveas != '':
113     default = saveas
114     self.downloadTo = abspath(default)
115     return default
116 carpaski 1.1
117 carpaski 1.2 def newpath(self, path):
118     self.downloadTo = path
119 carpaski 1.1
120     def prefix_array(array,prefix,doblanks=1):
121 carpaski 1.2 """Prepends a given prefix to each element in an Array/List/Tuple.
122     Returns a List."""
123     if type(array) not in [types.ListType, types.TupleType]:
124     raise TypeError, "List or Tuple expected. Got %s" % type(array)
125     newarray=[]
126     for x in array:
127     if x or doblanks:
128     newarray.append(prefix + x)
129     else:
130     newarray.append(x)
131     return newarray
132 carpaski 1.1
133     def list_files(dirlist):
134     file_list = []
135     for mydir in dirlist:
136     if isdir(mydir):
137     try:
138     file_list += list_files(prefix_array(listdir(mydir),mydir+"/"))
139     except Exception,e:
140     print e
141     elif isfile(mydir):
142     file_list.append(mydir)
143     else:
144     print "Unknown type:",mydir
145     return file_list
146    
147    
148     def progress(amount):
149     # Dummy function for the create-torrent progress meter
150     pass
151    
152    
153 carpaski 1.5 def writemsg(msg,fileName,overwrite=0,create=0,echo=1):
154 carpaski 1.2 if overwrite or create:
155     fileHandle = open(fileName, "w")
156     else:
157     fileHandle = open(fileName, "a")
158    
159     fileHandle.write(msg+"\n")
160     fileHandle.flush()
161     fileHandle.close()
162 carpaski 1.5 if echo:
163     print msg
164 carpaski 1.1
165 carpaski 1.4
166     # Useless variables
167     cols = 80
168    
169     # This is 2^x ... put the x below -- Chunk size for sha1 -- 18 == 256k
170 carpaski 1.5 piece_size_pow2 = 18
171     max_starting_threads = 1
172     loop_mod = 4320 # Rotating value (for logs)
173     rotate_logs = 0 # Make one log or roate into loop_mod logs
174     cycle_sleep = 20
175     spawn_thread_delay = 0
176     debug = 0
177     minimum_age = 30
178     port_min = 6881
179     port_max = 7280
180     check_hashes = 0
181     shuffle_files = 1
182     master_seed = 0
183     min_uploads = 20
184     max_uploads = 20
185     download_slice_size = 65536
186     max_slice_length = 131072
187     request_backlog = 20
188     timeout = 90
189     timeout_check_interval = 20
190     stats = 1
191 carpaski 1.4
192     # No trailing /
193     file_base = "/usr/portage/distfiles"
194     torrent_base = "/tmp/torrents"
195    
196     comment = "Gentoo Linux BitTorrent Mirror System"
197     tracker = "http://egret.gentoo.org:6969/announce"
198    
199 carpaski 1.1 if __name__ == '__main__':
200 carpaski 1.5 try:
201 carpaski 1.4 mydict = getconfig("bt_daemon.conf")
202     if not mydict:
203     print "bt_daemon.conf is missing"
204 carpaski 1.5
205     if mydict.has_key("file_base") and mydict["file_base"]:
206     file_base = os.path.normpath(mydict["file_base"])
207     else:
208     print "file_base not specified in config."
209 carpaski 1.4 sys.exit(1)
210 carpaski 1.5
211     if mydict.has_key("torrent_base") and mydict["torrent_base"]:
212     torrent_base = os.path.normpath(mydict["torrent_base"])
213 carpaski 1.4 else:
214 carpaski 1.5 print "torrent_base not specified in config."
215 carpaski 1.4 sys.exit(1)
216 carpaski 1.5
217     threads = {}
218     dirs = [torrent_base[:],file_base[:]]
219     re_file = re.compile("("+torrent_base[:]+"|"+file_base[:]+")(/.+?)(\.torrent)?$")
220     absmax_upRate = 0.0
221     absmax_upFile = ''
222    
223     loop_val = 0
224     while(True):
225     mydict = getconfig("bt_daemon.conf")
226     if not mydict:
227     sys.stderr.write("bt_daemon.conf is missing or contains errors\n")
228    
229     if mydict.has_key("master_seed"):
230     master_seed = int(mydict["master_seed"])
231    
232     if mydict.has_key("piece_size_pow2"):
233     piece_size_pow2 = int(mydict["piece_size_pow2"])
234     if mydict.has_key("max_starting_threads"):
235     max_starting_threads = int(mydict["max_starting_threads"])
236     if mydict.has_key("loop_mod"):
237     loop_mod = int(mydict["loop_mod"])
238     if mydict.has_key("rotate_logs"):
239     rotate_logs = int(mydict["rotate_logs"])
240     if mydict.has_key("cycle_sleep"):
241     cycle_sleep = int(mydict["cycle_sleep"])
242     if mydict.has_key("spawn_thread_delay"):
243     spawn_thread_delay = int(mydict["spawn_thread_delay"])
244     if mydict.has_key("debug"):
245     debug = int(mydict["debug"])
246    
247     if mydict.has_key("min_uploads"):
248     min_uploads = int(mydict["min_uploads"])
249     if mydict.has_key("max_uploads"):
250     max_uploads = int(mydict["max_uploads"])
251     if mydict.has_key("download_slice_size"):
252     download_slice_size = int(mydict["download_slice_size"])
253     if mydict.has_key("max_slice_length"):
254     max_slice_length = int(mydict["max_slice_length"])
255     if mydict.has_key("request_backlog"):
256     request_backlog = int(mydict["request_backlog"])
257     if mydict.has_key("timeout"):
258     timeout = int(mydict["timeout"])
259     if mydict.has_key("timeout_check_interval"):
260     timeout_check_interval = int(mydict["timeout_check_interval"])
261    
262     if mydict.has_key("comment"):
263     comment = mydict["comment"]
264     if mydict.has_key("port_min"):
265     port_min = mydict["port_min"]
266     if mydict.has_key("port_max"):
267     port_max = mydict["port_max"]
268     if mydict.has_key("check_hashes"):
269     check_hashes = mydict["check_hashes"]
270    
271     if mydict.has_key("tracker") and mydict["tracker"]:
272     tracker = mydict["tracker"]
273 carpaski 1.1 else:
274 carpaski 1.5 print "tracker not specified in config."
275     sys.exit(1)
276    
277    
278     loop_val = (loop_val + 1) % loop_mod
279     files = list_files(dirs)
280     files.sort()
281     new_files = []
282     torrents = []
283    
284     if shuffle_files:
285     random.shuffle(files)
286    
287     for f in files:
288     try:
289     match = re_file.match(f)
290     if not match:
291     print "Failed to match:",f
292     else:
293     if match.group(1) == torrent_base:
294     if match.group(3): # It's a torrent, yay!
295     if match.group(2) not in torrents:
296     mystat = stat(f)
297     # only add if it has a size and is not actively modified.
298     if mystat[ST_SIZE]>0 and (time()-mystat[ST_MTIME])>minimum_age:
299     torrents.append(match.group(2))
300     while(match.group(2) in new_files):
301     del new_files[new_files.index(match.group(2))]
302     elif torrent_base != file_base and debug:
303     print "Not a torrent file:",f
304     if match.group(1) == file_base:
305     if match.group(3) and torrent_base != file_base and debug:
306     print "Torrent in file directory:",f
307     elif match.group(2) not in torrents:
308     mystat = stat(f)
309     if mystat[ST_SIZE]>0 and (time()-mystat[ST_MTIME])>minimum_age:
310     new_files.append(match.group(2))
311     if debug:
312     print "New file:",f
313     else:
314     if debug:
315     print "Actively modified: ",f
316     except Exception,e:
317     traceback.print_last()
318     print "--- IGNORING FILE:",t
319     print
320    
321     for t in threads.keys():
322     # Remove threads that we don't have a bittorrent for.
323     if t not in torrents or not os.path.exists(file_base+t):
324     print "Stopping torrent [%s]: %s" % (ctime(),t)
325     threads[t][0]._Thread__stop()
326     del threads[t]
327     while(t in torrents):
328     del torrents[torrents.index(t)]
329    
330     elif not threads[t][0].isAlive():
331     print "Thread died [%s]: %s" % (ctime(),t)
332     del threads[t]
333    
334     completed = []
335     fetching = []
336     hashing = []
337     stalled = []
338    
339     stats_up = []
340     stats_down = []
341    
342     totalup = 0
343     rateup = 0
344     ratedown = 0
345    
346     max_up_rate = 0
347     max_up_file = ''
348     max_down_rate = 0
349     max_down_file = ''
350     completed_string = ""
351     for t in threads.keys():
352     # Let's do some stats.
353     if not threads[t][1].failed and threads[t][0].isAlive():
354     if threads[t][1].done:
355     completed.append(t)
356     else:
357     fetching.append(t)
358     if not threads[t][1].active:
359     hashing.append(t)
360     elif threads[t][1].downRate == 0:
361     stalled.append(t)
362    
363     stats_up.append("%s\tT:%f\tR:%03.2f\t%s" % (t,threads[t][1].upTotal,threads[t][1].upRate,threads[t][1].activity))
364     stats_down.append("%s\tT:%f\tR:%03.2f\t%s" % (t,threads[t][1].percentDone,threads[t][1].downRate,threads[t][1].activity))
365    
366     totalup += threads[t][1].upTotal
367    
368     rateup += threads[t][1].upRate
369     if threads[t][1].upRate > max_up_rate:
370     max_up_rate = threads[t][1].upRate
371     max_up_file = threads[t][1].file[:]
372     if threads[t][1].upRate > absmax_upRate:
373     absmax_upRate = threads[t][1].upRate
374     absmax_upFile = threads[t][1].file[:]
375    
376     ratedown += threads[t][1].downRate
377     if threads[t][1].downRate > max_down_rate:
378     max_down_rate = threads[t][1].downRate
379     max_down_file = threads[t][1].file[:]
380    
381     if stats:
382     if rotate_logs:
383     filename = "bt_daemon-%d-%04d.stats" % (os.getpid(),loop_val)
384     makenew = 1
385 carpaski 1.2 else:
386 carpaski 1.5 filename = "bt_daemon-%d.stats" % (os.getpid())
387     makenew = 0
388     writemsg("",filename,overwrite=makenew)
389     writemsg("Stats: %s" % (ctime()) ,filename)
390     writemsg("Seeding: %d" % (len(completed)) ,filename)
391     writemsg("Fetching: %d (stalled: %d) (hashing: %d)" % (len(fetching),len(stalled),len(hashing)),filename)
392     writemsg("Total up: %.3f MB" % (totalup) ,filename)
393     writemsg("UpRate: %.2f k/s" % (rateup) ,filename)
394     writemsg("UpMaxNow: %.2f k/s '%s'" % (max_up_rate,max_up_file) ,filename)
395     writemsg("UpAbsMax: %.2f k/s '%s'" % (absmax_upRate,absmax_upFile) ,filename)
396     writemsg("DownRate: %.2f k/s" % (ratedown) ,filename)
397     writemsg("DownBest: %.2f k/s '%s'" % (max_down_rate,max_down_file) ,filename)
398     writemsg("",filename)
399 carpaski 1.2
400 carpaski 1.5 completed.sort()
401     writemsg(string.join(completed, "\n"), filename+".completed", overwrite=1, echo=0)
402     hashing.sort()
403     writemsg(string.join(hashing, "\n"), filename+".hashing", overwrite=1, echo=0)
404     stalled.sort()
405     writemsg(string.join(stalled, "\n"), filename+".stalled", overwrite=1, echo=0)
406     stats_up.sort()
407     writemsg(string.join(stats_up, "\n"), filename+".stats_up", overwrite=1, echo=0)
408     stats_down.sort()
409     writemsg(string.join(stats_down, "\n"), filename+".stats_down", overwrite=1, echo=0)
410    
411     for t in torrents:
412     # Start threads up if we have load free for them. Startup is harsh.
413     if t not in threads.keys() and (max_starting_threads - len(hashing) > 0):
414     hashing.append(t)
415     print "Starting torrent [%s]: %s" % (ctime(),t)
416     h = HeadlessDisplayer()
417     try:
418     if not os.path.isdir(file_base+"/"+os.path.dirname(t)):
419     os.makedirs(file_base+"/"+os.path.dirname(t))
420     download_args = [
421     # responsefile -> Torrent filename
422     '--responsefile', torrent_base+"/"+t+".torrent",
423     # saveas -> Real filename
424     '--saveas', file_base+"/"+t,
425     # max_slice_length -> How much we can send in a single request
426     '--max_slice_length', str(max_slice_length),
427     # timeout_check_interval -> Time between timeout checks
428     '--timeout_check_interval', str(timeout_check_interval),
429     # timeout -> How long until we time out a connection
430     '--timeout', str(timeout),
431     # minport -> Lowest port allowed for files
432     '--minport', str(port_min),
433     # maxport -> Highest port allowed for files
434     '--maxport', str(port_max),
435     # request_backlog -> How many requests per pipe do we hold
436     '--request_backlog', str(request_backlog),
437     # download_slice_size -> How much do we request?
438     '--download_slice_size', str(download_slice_size),
439     # max_uploads -> How many do we serve out?
440     '--max_uploads', str(max_uploads),
441     # min_uploads -> When do we switch from random to rare?
442     '--min_uploads', str(min_uploads),
443     # check_hashes -> Do we check the on-disk files before joining?
444     '--check_hashes', str(check_hashes)
445     ]
446     t_thread = Thread(target=download,name="BitTorrent::"+t,args=(download_args, h.chooseFile, h.display, h.finished, h.error, Event(), cols))
447     threads[t] = [t_thread,h]
448     t_thread.start()
449     sleep(spawn_thread_delay)
450     except Exception, e:
451     print "Failed to start thread: %s" % (t)
452     print "Reason: %s" % (str(e))
453    
454     if master_seed:
455     for f in new_files:
456     # Create the torrents for files that don't have them.
457     try:
458     print "Creating torrent [%s]: %s" % (ctime(),f)
459     if not os.path.isdir(torrent_base+"/"+os.path.dirname(f)):
460     os.makedirs(torrent_base+"/"+os.path.dirname(f))
461     make_meta_file(file_base+"/"+f,tracker,piece_size_pow2,progress=progress,comment=comment,target=torrent_base+"/"+f+".torrent")
462     except Exception, e:
463     print "Failed to create torrent: "+str(e)
464    
465     sleep(cycle_sleep)
466     except Exception, e:
467     import signal,traceback
468     print e
469     traceback.print_stack()
470     os.kill(0,signal.SIGTERM)

  ViewVC Help
Powered by ViewVC 1.1.20