/[gentoo-src]/bittorrent/bt_daemon.py
Gentoo

Contents of /bittorrent/bt_daemon.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.4 - (show annotations) (download) (as text)
Tue May 25 23:51:44 2004 UTC (9 years, 10 months ago) by carpaski
Branch: MAIN
Changes since 1.3: +98 -35 lines
File MIME type: text/x-python
New config file, removed dependence on modified bt installation, and
made it aware of changing files as a work around for atomicity problems.

1 #!/usr/bin/python -O
2 # $Id: bt_daemon.py,v 1.3 2004/05/02 01:41:19 carpaski Exp $
3 #
4 # Greatly modified by carpaski@gentoo.org
5 # btdownloadheadless written by Bram Cohen
6 # see LICENSE.txt for license information
7
8 import os,sys
9 os.nice(5)
10
11 sys.path = ["/usr/lib/portage/pym"] + sys.path + ["/usr/bin"]
12 from portage import getconfig
13
14 import BitTorrent
15 from BitTorrent.download import download
16 from threading import Event,Thread
17 import types
18 import re
19 from os import listdir, getcwd, stat
20 from stat import ST_SIZE,ST_MTIME
21 from os.path import abspath,isdir,isfile,normpath
22 from sys import argv, stdout, path
23 from cStringIO import StringIO
24 from time import time,ctime,sleep
25
26 from btmakemetafile import make_meta_file
27
28 def hours(n):
29 if n == -1:
30 return '<unknown>'
31 if n == 0:
32 return 'complete!'
33 n = long(n)
34 h, r = divmod(n, 60 * 60)
35 m, sec = divmod(r, 60)
36 if h > 1000000:
37 return '<unknown>'
38 if h > 0:
39 return '%d hour %02d min %02d sec' % (h, m, sec)
40 else:
41 return '%d min %02d sec' % (m, sec)
42
43 class HeadlessDisplayer:
44 def __init__(self):
45 self.done = False
46 self.failed = False
47 self.file = ''
48 self.activity = ''
49 self.percentDone = 0.0
50 self.timeEst = 0
51 self.downloadTo = ''
52 self.downRate = 0.0
53 self.upRate = 0.0
54 self.downTotal = 0.0
55 self.upTotal = 0
56 self.errors = []
57 self.last_update_time = 0
58
59 def finished(self):
60 self.done = True
61 self.percentDone = 100
62 self.timeEst = 0
63 self.downRate = 0
64 self.display({})
65
66 def failed(self):
67 self.done = True
68 self.failed = True
69 self.percentDone = 0
70 self.timeEst = -1
71 self.downRate = 0
72 self.display({})
73
74 def error(self, errormsg):
75 self.errors.append(errormsg)
76 self.display({})
77
78 def display(self, dict):
79 if ((self.last_update_time + 1) > time()):
80 if dict.get('fractionDone') not in (0.0, 1.0):
81 if not dict.has_key('activity'):
82 return
83 self.last_update_time = time()
84 if dict.has_key('fractionDone'):
85 self.percentDone = float(int(dict['fractionDone'] * 1000)) / 10
86 if dict.has_key('timeEst'):
87 self.timeEst = long(dict['timeEst'])
88 if dict.has_key('activity') and not self.done:
89 self.activity = dict['activity']
90 if dict.has_key('downRate'):
91 self.downRate = float(dict['downRate']) / (1 << 10)
92 if dict.has_key('upRate'):
93 self.upRate = float(dict['upRate']) / (1 << 10)
94 if dict.has_key('upTotal'):
95 self.upTotal = dict['upTotal']
96 if dict.has_key('downTotal'):
97 self.downTotal = dict['downTotal']
98 for x in self.errors:
99 print self.file+": "+str(x)
100 self.errors = []
101 return
102
103 def chooseFile(self, default, size, saveas, dir):
104 self.file = '%s (%.1f MB)' % (default, float(size) / (1 << 20))
105 if saveas != '':
106 default = saveas
107 self.downloadTo = abspath(default)
108 return default
109
110 def newpath(self, path):
111 self.downloadTo = path
112
113 def prefix_array(array,prefix,doblanks=1):
114 """Prepends a given prefix to each element in an Array/List/Tuple.
115 Returns a List."""
116 if type(array) not in [types.ListType, types.TupleType]:
117 raise TypeError, "List or Tuple expected. Got %s" % type(array)
118 newarray=[]
119 for x in array:
120 if x or doblanks:
121 newarray.append(prefix + x)
122 else:
123 newarray.append(x)
124 return newarray
125
126 def list_files(dirlist):
127 file_list = []
128 for mydir in dirlist:
129 if isdir(mydir):
130 try:
131 file_list += list_files(prefix_array(listdir(mydir),mydir+"/"))
132 except Exception,e:
133 print e
134 elif isfile(mydir):
135 file_list.append(mydir)
136 else:
137 print "Unknown type:",mydir
138 return file_list
139
140
141 def progress(amount):
142 # Dummy function for the create-torrent progress meter
143 pass
144
145
146 def writemsg(msg,fileName,overwrite=0, create=0):
147 if overwrite or create:
148 fileHandle = open(fileName, "w")
149 else:
150 fileHandle = open(fileName, "a")
151
152 fileHandle.write(msg+"\n")
153 fileHandle.flush()
154 fileHandle.close()
155 print msg
156
157
158 # Useless variables
159 cols = 80
160
161 # This is 2^x ... put the x below -- Chunk size for sha1 -- 18 == 256k
162 piece_size_pow2 = 18
163 max_starting_threads = 10
164 loop_mod = 4320 # Rotating value (for logs)
165 rotate_logs = 0 # Make one log or roate into loop_mod logs
166 cycle_sleep = 20
167 spawn_thread_delay = 0
168 debug = 0
169 minimum_age = 30
170 port_min = 6881
171 port_max = 7280
172 check_hashes = 0
173
174 # No trailing /
175 file_base = "/usr/portage/distfiles"
176 torrent_base = "/tmp/torrents"
177
178 comment = "Gentoo Linux BitTorrent Mirror System"
179 tracker = "http://egret.gentoo.org:6969/announce"
180
181 if __name__ == '__main__':
182 mydict = getconfig("bt_daemon.conf")
183 if not mydict:
184 print "bt_daemon.conf is missing"
185 sys.exit(1)
186
187 if mydict.has_key("file_base") and mydict["file_base"]:
188 file_base = os.path.normpath(mydict["file_base"])
189 else:
190 print "file_base not specified in config."
191 sys.exit(1)
192
193 if mydict.has_key("torrent_base") and mydict["torrent_base"]:
194 torrent_base = os.path.normpath(mydict["torrent_base"])
195 else:
196 print "torrent_base not specified in config."
197 sys.exit(1)
198
199 threads = {}
200 dirs = [torrent_base[:],file_base[:]]
201 re_file = re.compile("("+torrent_base[:]+"|"+file_base[:]+")(/.+?)(\.torrent)?$")
202 absmax_upRate = 0.0
203 absmax_upFile = ''
204
205 loop_val = 0
206 while(True):
207 mydict = getconfig("bt_daemon.conf")
208 if not mydict:
209 print "bt_daemon.conf is missing"
210 sys.exit(1)
211
212 if mydict.has_key("piece_size_pow2"):
213 piece_size_pow2 = int(mydict["piece_size_pow2"])
214 if mydict.has_key("max_starting_threads"):
215 max_starting_threads = int(mydict["max_starting_threads"])
216 if mydict.has_key("loop_mod"):
217 loop_mod = int(mydict["loop_mod"])
218 if mydict.has_key("rotate_logs"):
219 rotate_logs = int(mydict["rotate_logs"])
220 if mydict.has_key("cycle_sleep"):
221 cycle_sleep = int(mydict["cycle_sleep"])
222 if mydict.has_key("spawn_thread_delay"):
223 spawn_thread_delay = int(mydict["spawn_thread_delay"])
224 if mydict.has_key("debug"):
225 debug = int(mydict["debug"])
226
227 if mydict.has_key("comment"):
228 comment = mydict["comment"]
229 if mydict.has_key("port_min"):
230 port_min = mydict["port_min"]
231 if mydict.has_key("port_max"):
232 port_max = mydict["port_max"]
233 if mydict.has_key("check_hashes"):
234 check_hashes = mydict["check_hashes"]
235
236 if mydict.has_key("tracker") and mydict["tracker"]:
237 tracker = mydict["tracker"]
238 else:
239 print "tracker not specified in config."
240 sys.exit(1)
241
242
243 loop_val = (loop_val + 1) % loop_mod
244 files = list_files(dirs)
245 files.sort()
246 new_files = []
247 torrents = []
248
249 for f in files:
250 match = re_file.match(f)
251 if not match:
252 print "Failed to match:",f
253 else:
254 if match.group(1) == torrent_base:
255 if match.group(3): # It's a torrent, yay!
256 if match.group(2) not in torrents:
257 mystat = stat(f)
258 # only add if it has a size and is not actively modified.
259 if mystat[ST_SIZE]>0 and (time()-mystat[ST_MTIME])>minimum_age:
260 torrents.append(match.group(2))
261 while(match.group(2) in new_files):
262 del new_files[new_files.index(match.group(2))]
263 elif torrent_base != file_base and debug:
264 print "Not a torrent file:",f
265 if match.group(1) == file_base:
266 if match.group(3) and torrent_base != file_base and debug:
267 print "Torrent in file directory:",f
268 elif match.group(2) not in torrents:
269 if debug:
270 print "New file:",f
271 mystat = stat(f)
272 if mystat[ST_SIZE]>0 and (time()-mystat[ST_MTIME])>minimum_age:
273 new_files.append(match.group(2))
274
275 for t in threads.keys():
276 # Remove threads that we don't have a bittorrent for.
277 if t not in torrents or not os.path.exists(file_base+t):
278 print "Stopping torrent [%s]: %s" % (ctime(),t)
279 threads[t][0]._Thread__stop()
280 del threads[t]
281 while(t in torrents):
282 del torrents[torrents.index(t)]
283
284 elif not threads[t][0].isAlive():
285 print "Thread died [%s]: %s" % (ctime(),t)
286 del threads[t]
287
288 fetching = 0
289 completed = 0
290 stalled = 0
291 totalup = 0
292 rateup = 0
293 ratedown = 0
294
295 max_up_rate = 0
296 max_up_file = ''
297 max_down_rate = 0
298 max_down_file = ''
299 for t in threads.keys():
300 # Let's do some stats.
301 if not threads[t][1].failed and threads[t][0].isAlive():
302 if threads[t][1].done:
303 completed += 1
304 else:
305 fetching += 1
306 if threads[t][1].downRate == 0:
307 stalled += 1
308
309 totalup += threads[t][1].upTotal
310
311 rateup += threads[t][1].upRate
312 if threads[t][1].upRate > max_up_rate:
313 max_up_rate = threads[t][1].upRate
314 max_up_file = threads[t][1].file[:]
315 if threads[t][1].upRate > absmax_upRate:
316 absmax_upRate = threads[t][1].upRate
317 absmax_upFile = threads[t][1].file[:]
318
319 ratedown += threads[t][1].downRate
320 if threads[t][1].downRate > max_down_rate:
321 max_down_rate = threads[t][1].downRate
322 max_down_file = threads[t][1].file[:]
323
324 if os.path.exists(".stats"):
325 if ROTATE_LOGS:
326 filename = "bt_daemon-%d-%04d.stats" % (os.getpid(),loop_val)
327 makenew = 1
328 else:
329 filename = "bt_daemon-%d.stats" % (os.getpid())
330 makenew = 0
331 writemsg("",filename,overwrite=makenew)
332 writemsg("Stats: %s" % (ctime()) ,filename)
333 writemsg("Seeding: %d" % (completed) ,filename)
334 writemsg("Fetching: %d(%d)" % (fetching,stalled) ,filename)
335 writemsg("Total up: %.3f MB" % (totalup) ,filename)
336 writemsg("UpRate: %.2f k/s" % (rateup) ,filename)
337 writemsg("UpMaxNow: %.2f k/s '%s'" % (max_up_rate,max_up_file) ,filename)
338 writemsg("UpAbsMax: %.2f k/s '%s'" % (absmax_upRate,absmax_upFile) ,filename)
339 writemsg("DownRate: %.2f k/s" % (ratedown) ,filename)
340 writemsg("DownBest: %.2f k/s '%s'" % (max_down_rate,max_down_file) ,filename)
341 writemsg("",filename)
342
343 for t in torrents:
344 # Start threads up if we have load free for them. Startup is harsh.
345 if t not in threads.keys() and ((len(threads.keys())-completed) < max_starting_threads):
346 print "Starting torrent [%s]: %s" % (ctime(),t)
347 h = HeadlessDisplayer()
348 try:
349 if not os.path.isdir(file_base+"/"+os.path.dirname(t)):
350 os.makedirs(file_base+"/"+os.path.dirname(t))
351 download_args = [
352 # responsefile -> Torrent filename
353 '--responsefile', torrent_base+"/"+t+".torrent",
354 # saveas -> Real filename
355 '--saveas', file_base+"/"+t,
356 # max_slice_length -> How much we can get in a single request
357 '--max_slice_length', "65536",
358 # timeout_check_interval -> Time between timeout checks
359 '--timeout_check_interval', '30',
360 # timeout -> How long until we time out a connection
361 '--timeout', '120',
362 # minport -> Lowest port allowed for files
363 '--minport', str(port_min),
364 # maxport -> Highest port allowed for files
365 '--maxport', str(port_max),
366 # request_backlog -> How many requests per pipe do we hold
367 '--request_backlog', '20',
368 # download_slice_size -> How much do we request?
369 '--download_slice_size', '65536',
370 # max_uploads -> How many do we serve out?
371 '--max_uploads', '20',
372 # check_hashes -> Do we check the on-disk files before joining?
373 '--check_hashes', str(check_hashes)
374 ]
375 t_thread = Thread(target=download,name="BitTorrent::"+t,args=(download_args, h.chooseFile, h.display, h.finished, h.error, Event(), cols))
376 threads[t] = [t_thread,h]
377 t_thread.start()
378 sleep(spawn_thread_delay)
379 except Exception, e:
380 print "Failed to start thread: %s" % (t)
381 print "Reason: %s" % (str(e))
382
383 for f in new_files:
384 # Create the torrents for files that don't have them.
385 try:
386 print "Creating torrent [%s]: %s" % (ctime(),f)
387 if not os.path.isdir(torrent_base+"/"+os.path.dirname(f)):
388 os.makedirs(torrent_base+"/"+os.path.dirname(f))
389 make_meta_file(file_base+"/"+f,tracker,piece_size_pow2,progress=progress,comment=comment,target=torrent_base+"/"+f+".torrent")
390 except Exception, e:
391 print "Failed to create torrent: "+str(e)
392
393 sleep(cycle_sleep)

  ViewVC Help
Powered by ViewVC 1.1.20