/[gentoo-src]/bittorrent/bt_daemon.py
Gentoo

Contents of /bittorrent/bt_daemon.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.4 - (hide annotations) (download) (as text)
Tue May 25 23:51:44 2004 UTC (9 years, 10 months ago) by carpaski
Branch: MAIN
Changes since 1.3: +98 -35 lines
File MIME type: text/x-python
New config file, removed dependence on modified bt installation, and
made it aware of changing files as a work around for atomicity problems.

1 carpaski 1.1 #!/usr/bin/python -O
2 carpaski 1.4 # $Id: bt_daemon.py,v 1.3 2004/05/02 01:41:19 carpaski Exp $
3     #
4 carpaski 1.1 # Greatly modified by carpaski@gentoo.org
5     # btdownloadheadless written by Bram Cohen
6     # see LICENSE.txt for license information
7    
8 carpaski 1.4 import os,sys
9 carpaski 1.2 os.nice(5)
10 carpaski 1.1
11 carpaski 1.4 sys.path = ["/usr/lib/portage/pym"] + sys.path + ["/usr/bin"]
12     from portage import getconfig
13 carpaski 1.2
14 carpaski 1.1 import BitTorrent
15     from BitTorrent.download import download
16     from threading import Event,Thread
17     import types
18     import re
19     from os import listdir, getcwd, stat
20 carpaski 1.4 from stat import ST_SIZE,ST_MTIME
21 carpaski 1.1 from os.path import abspath,isdir,isfile,normpath
22     from sys import argv, stdout, path
23     from cStringIO import StringIO
24     from time import time,ctime,sleep
25    
26 carpaski 1.4 from btmakemetafile import make_meta_file
27 carpaski 1.1
28     def hours(n):
29 carpaski 1.2 if n == -1:
30     return '<unknown>'
31     if n == 0:
32     return 'complete!'
33     n = long(n)
34     h, r = divmod(n, 60 * 60)
35     m, sec = divmod(r, 60)
36     if h > 1000000:
37     return '<unknown>'
38     if h > 0:
39     return '%d hour %02d min %02d sec' % (h, m, sec)
40     else:
41     return '%d min %02d sec' % (m, sec)
42 carpaski 1.1
43     class HeadlessDisplayer:
44 carpaski 1.2 def __init__(self):
45     self.done = False
46     self.failed = False
47     self.file = ''
48     self.activity = ''
49     self.percentDone = 0.0
50     self.timeEst = 0
51     self.downloadTo = ''
52     self.downRate = 0.0
53     self.upRate = 0.0
54     self.downTotal = 0.0
55     self.upTotal = 0
56     self.errors = []
57     self.last_update_time = 0
58    
59     def finished(self):
60     self.done = True
61     self.percentDone = 100
62     self.timeEst = 0
63     self.downRate = 0
64     self.display({})
65    
66     def failed(self):
67     self.done = True
68     self.failed = True
69     self.percentDone = 0
70     self.timeEst = -1
71     self.downRate = 0
72     self.display({})
73    
74     def error(self, errormsg):
75     self.errors.append(errormsg)
76     self.display({})
77    
78     def display(self, dict):
79     if ((self.last_update_time + 1) > time()):
80     if dict.get('fractionDone') not in (0.0, 1.0):
81     if not dict.has_key('activity'):
82     return
83     self.last_update_time = time()
84     if dict.has_key('fractionDone'):
85     self.percentDone = float(int(dict['fractionDone'] * 1000)) / 10
86     if dict.has_key('timeEst'):
87     self.timeEst = long(dict['timeEst'])
88     if dict.has_key('activity') and not self.done:
89     self.activity = dict['activity']
90     if dict.has_key('downRate'):
91     self.downRate = float(dict['downRate']) / (1 << 10)
92     if dict.has_key('upRate'):
93     self.upRate = float(dict['upRate']) / (1 << 10)
94     if dict.has_key('upTotal'):
95     self.upTotal = dict['upTotal']
96     if dict.has_key('downTotal'):
97     self.downTotal = dict['downTotal']
98     for x in self.errors:
99     print self.file+": "+str(x)
100     self.errors = []
101     return
102    
103     def chooseFile(self, default, size, saveas, dir):
104     self.file = '%s (%.1f MB)' % (default, float(size) / (1 << 20))
105     if saveas != '':
106     default = saveas
107     self.downloadTo = abspath(default)
108     return default
109 carpaski 1.1
110 carpaski 1.2 def newpath(self, path):
111     self.downloadTo = path
112 carpaski 1.1
113     def prefix_array(array,prefix,doblanks=1):
114 carpaski 1.2 """Prepends a given prefix to each element in an Array/List/Tuple.
115     Returns a List."""
116     if type(array) not in [types.ListType, types.TupleType]:
117     raise TypeError, "List or Tuple expected. Got %s" % type(array)
118     newarray=[]
119     for x in array:
120     if x or doblanks:
121     newarray.append(prefix + x)
122     else:
123     newarray.append(x)
124     return newarray
125 carpaski 1.1
126     def list_files(dirlist):
127     file_list = []
128     for mydir in dirlist:
129     if isdir(mydir):
130     try:
131     file_list += list_files(prefix_array(listdir(mydir),mydir+"/"))
132     except Exception,e:
133     print e
134     elif isfile(mydir):
135     file_list.append(mydir)
136     else:
137     print "Unknown type:",mydir
138     return file_list
139    
140    
141     def progress(amount):
142     # Dummy function for the create-torrent progress meter
143     pass
144    
145    
146 carpaski 1.2 def writemsg(msg,fileName,overwrite=0, create=0):
147     if overwrite or create:
148     fileHandle = open(fileName, "w")
149     else:
150     fileHandle = open(fileName, "a")
151    
152     fileHandle.write(msg+"\n")
153     fileHandle.flush()
154     fileHandle.close()
155     print msg
156 carpaski 1.1
157 carpaski 1.4
158     # Useless variables
159     cols = 80
160    
161     # This is 2^x ... put the x below -- Chunk size for sha1 -- 18 == 256k
162     piece_size_pow2 = 18
163     max_starting_threads = 10
164     loop_mod = 4320 # Rotating value (for logs)
165     rotate_logs = 0 # Make one log or roate into loop_mod logs
166     cycle_sleep = 20
167     spawn_thread_delay = 0
168     debug = 0
169     minimum_age = 30
170     port_min = 6881
171     port_max = 7280
172     check_hashes = 0
173    
174     # No trailing /
175     file_base = "/usr/portage/distfiles"
176     torrent_base = "/tmp/torrents"
177    
178     comment = "Gentoo Linux BitTorrent Mirror System"
179     tracker = "http://egret.gentoo.org:6969/announce"
180    
181 carpaski 1.1 if __name__ == '__main__':
182 carpaski 1.4 mydict = getconfig("bt_daemon.conf")
183     if not mydict:
184     print "bt_daemon.conf is missing"
185     sys.exit(1)
186    
187     if mydict.has_key("file_base") and mydict["file_base"]:
188     file_base = os.path.normpath(mydict["file_base"])
189     else:
190     print "file_base not specified in config."
191     sys.exit(1)
192    
193     if mydict.has_key("torrent_base") and mydict["torrent_base"]:
194     torrent_base = os.path.normpath(mydict["torrent_base"])
195     else:
196     print "torrent_base not specified in config."
197     sys.exit(1)
198 carpaski 1.1
199     threads = {}
200     dirs = [torrent_base[:],file_base[:]]
201     re_file = re.compile("("+torrent_base[:]+"|"+file_base[:]+")(/.+?)(\.torrent)?$")
202 carpaski 1.2 absmax_upRate = 0.0
203     absmax_upFile = ''
204 carpaski 1.1
205 carpaski 1.2 loop_val = 0
206 carpaski 1.1 while(True):
207 carpaski 1.4 mydict = getconfig("bt_daemon.conf")
208     if not mydict:
209     print "bt_daemon.conf is missing"
210     sys.exit(1)
211    
212     if mydict.has_key("piece_size_pow2"):
213     piece_size_pow2 = int(mydict["piece_size_pow2"])
214     if mydict.has_key("max_starting_threads"):
215     max_starting_threads = int(mydict["max_starting_threads"])
216     if mydict.has_key("loop_mod"):
217     loop_mod = int(mydict["loop_mod"])
218     if mydict.has_key("rotate_logs"):
219     rotate_logs = int(mydict["rotate_logs"])
220     if mydict.has_key("cycle_sleep"):
221     cycle_sleep = int(mydict["cycle_sleep"])
222     if mydict.has_key("spawn_thread_delay"):
223     spawn_thread_delay = int(mydict["spawn_thread_delay"])
224     if mydict.has_key("debug"):
225     debug = int(mydict["debug"])
226    
227     if mydict.has_key("comment"):
228     comment = mydict["comment"]
229     if mydict.has_key("port_min"):
230     port_min = mydict["port_min"]
231     if mydict.has_key("port_max"):
232     port_max = mydict["port_max"]
233     if mydict.has_key("check_hashes"):
234     check_hashes = mydict["check_hashes"]
235    
236     if mydict.has_key("tracker") and mydict["tracker"]:
237     tracker = mydict["tracker"]
238     else:
239     print "tracker not specified in config."
240     sys.exit(1)
241    
242    
243     loop_val = (loop_val + 1) % loop_mod
244 carpaski 1.1 files = list_files(dirs)
245     files.sort()
246     new_files = []
247     torrents = []
248    
249     for f in files:
250     match = re_file.match(f)
251     if not match:
252     print "Failed to match:",f
253     else:
254     if match.group(1) == torrent_base:
255     if match.group(3): # It's a torrent, yay!
256 carpaski 1.4 if match.group(2) not in torrents:
257     mystat = stat(f)
258     # only add if it has a size and is not actively modified.
259     if mystat[ST_SIZE]>0 and (time()-mystat[ST_MTIME])>minimum_age:
260     torrents.append(match.group(2))
261 carpaski 1.1 while(match.group(2) in new_files):
262     del new_files[new_files.index(match.group(2))]
263     elif torrent_base != file_base and debug:
264     print "Not a torrent file:",f
265     if match.group(1) == file_base:
266     if match.group(3) and torrent_base != file_base and debug:
267     print "Torrent in file directory:",f
268     elif match.group(2) not in torrents:
269     if debug:
270     print "New file:",f
271 carpaski 1.4 mystat = stat(f)
272     if mystat[ST_SIZE]>0 and (time()-mystat[ST_MTIME])>minimum_age:
273     new_files.append(match.group(2))
274 carpaski 1.1
275     for t in threads.keys():
276     # Remove threads that we don't have a bittorrent for.
277 carpaski 1.2 if t not in torrents or not os.path.exists(file_base+t):
278 carpaski 1.1 print "Stopping torrent [%s]: %s" % (ctime(),t)
279     threads[t][0]._Thread__stop()
280     del threads[t]
281     while(t in torrents):
282     del torrents[torrents.index(t)]
283    
284     elif not threads[t][0].isAlive():
285     print "Thread died [%s]: %s" % (ctime(),t)
286     del threads[t]
287 carpaski 1.2
288     fetching = 0
289     completed = 0
290     stalled = 0
291     totalup = 0
292     rateup = 0
293     ratedown = 0
294    
295     max_up_rate = 0
296     max_up_file = ''
297     max_down_rate = 0
298     max_down_file = ''
299     for t in threads.keys():
300     # Let's do some stats.
301     if not threads[t][1].failed and threads[t][0].isAlive():
302     if threads[t][1].done:
303     completed += 1
304     else:
305     fetching += 1
306     if threads[t][1].downRate == 0:
307     stalled += 1
308    
309     totalup += threads[t][1].upTotal
310    
311     rateup += threads[t][1].upRate
312     if threads[t][1].upRate > max_up_rate:
313     max_up_rate = threads[t][1].upRate
314     max_up_file = threads[t][1].file[:]
315     if threads[t][1].upRate > absmax_upRate:
316     absmax_upRate = threads[t][1].upRate
317     absmax_upFile = threads[t][1].file[:]
318    
319     ratedown += threads[t][1].downRate
320     if threads[t][1].downRate > max_down_rate:
321     max_down_rate = threads[t][1].downRate
322     max_down_file = threads[t][1].file[:]
323    
324     if os.path.exists(".stats"):
325     if ROTATE_LOGS:
326 carpaski 1.3 filename = "bt_daemon-%d-%04d.stats" % (os.getpid(),loop_val)
327 carpaski 1.2 makenew = 1
328     else:
329     filename = "bt_daemon-%d.stats" % (os.getpid())
330     makenew = 0
331     writemsg("",filename,overwrite=makenew)
332     writemsg("Stats: %s" % (ctime()) ,filename)
333     writemsg("Seeding: %d" % (completed) ,filename)
334     writemsg("Fetching: %d(%d)" % (fetching,stalled) ,filename)
335     writemsg("Total up: %.3f MB" % (totalup) ,filename)
336     writemsg("UpRate: %.2f k/s" % (rateup) ,filename)
337     writemsg("UpMaxNow: %.2f k/s '%s'" % (max_up_rate,max_up_file) ,filename)
338 carpaski 1.3 writemsg("UpAbsMax: %.2f k/s '%s'" % (absmax_upRate,absmax_upFile) ,filename)
339 carpaski 1.2 writemsg("DownRate: %.2f k/s" % (ratedown) ,filename)
340     writemsg("DownBest: %.2f k/s '%s'" % (max_down_rate,max_down_file) ,filename)
341     writemsg("",filename)
342 carpaski 1.1
343     for t in torrents:
344 carpaski 1.2 # Start threads up if we have load free for them. Startup is harsh.
345 carpaski 1.4 if t not in threads.keys() and ((len(threads.keys())-completed) < max_starting_threads):
346 carpaski 1.1 print "Starting torrent [%s]: %s" % (ctime(),t)
347     h = HeadlessDisplayer()
348     try:
349     if not os.path.isdir(file_base+"/"+os.path.dirname(t)):
350     os.makedirs(file_base+"/"+os.path.dirname(t))
351 carpaski 1.3 download_args = [
352     # responsefile -> Torrent filename
353 carpaski 1.4 '--responsefile', torrent_base+"/"+t+".torrent",
354 carpaski 1.3 # saveas -> Real filename
355     '--saveas', file_base+"/"+t,
356     # max_slice_length -> How much we can get in a single request
357     '--max_slice_length', "65536",
358     # timeout_check_interval -> Time between timeout checks
359     '--timeout_check_interval', '30',
360     # timeout -> How long until we time out a connection
361     '--timeout', '120',
362     # minport -> Lowest port allowed for files
363 carpaski 1.4 '--minport', str(port_min),
364 carpaski 1.3 # maxport -> Highest port allowed for files
365 carpaski 1.4 '--maxport', str(port_max),
366 carpaski 1.3 # request_backlog -> How many requests per pipe do we hold
367     '--request_backlog', '20',
368     # download_slice_size -> How much do we request?
369     '--download_slice_size', '65536',
370     # max_uploads -> How many do we serve out?
371     '--max_uploads', '20',
372     # check_hashes -> Do we check the on-disk files before joining?
373 carpaski 1.4 '--check_hashes', str(check_hashes)
374 carpaski 1.3 ]
375     t_thread = Thread(target=download,name="BitTorrent::"+t,args=(download_args, h.chooseFile, h.display, h.finished, h.error, Event(), cols))
376 carpaski 1.1 threads[t] = [t_thread,h]
377     t_thread.start()
378 carpaski 1.2 sleep(spawn_thread_delay)
379 carpaski 1.1 except Exception, e:
380     print "Failed to start thread: %s" % (t)
381     print "Reason: %s" % (str(e))
382    
383     for f in new_files:
384     # Create the torrents for files that don't have them.
385     try:
386     print "Creating torrent [%s]: %s" % (ctime(),f)
387     if not os.path.isdir(torrent_base+"/"+os.path.dirname(f)):
388     os.makedirs(torrent_base+"/"+os.path.dirname(f))
389 carpaski 1.4 make_meta_file(file_base+"/"+f,tracker,piece_size_pow2,progress=progress,comment=comment,target=torrent_base+"/"+f+".torrent")
390 carpaski 1.1 except Exception, e:
391     print "Failed to create torrent: "+str(e)
392    
393 carpaski 1.2 sleep(cycle_sleep)

  ViewVC Help
Powered by ViewVC 1.1.20