/[gentoo-src]/bittorrent/bt_daemon.py
Gentoo

Contents of /bittorrent/bt_daemon.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.3 - (show annotations) (download) (as text)
Sun May 2 01:41:19 2004 UTC (10 years, 4 months ago) by carpaski
Branch: MAIN
Changes since 1.2: +29 -4 lines
File MIME type: text/x-python
More fixes. More control on the client processes.

1 #!/usr/bin/python -O
2 # $Id: $
3
4 # Greatly modified by carpaski@gentoo.org
5 # btdownloadheadless written by Bram Cohen
6 # see LICENSE.txt for license information
7
8 import os
9 os.nice(5)
10
11 # This is 2^x ... put the x below -- Chunk size for sha1 -- 18 == 256k
12 piece_size_pow2 = 18
13 MAX_STARTING_THREADS = 10
14 LOOP_MOD = 4320 # Rotating value (for logs)
15 ROTATE_LOGS = 0 # Make one log or roate into loop_mod logs
16 cycle_sleep = 20
17 spawn_thread_delay = 0
18 debug = 0
19
20 # No trailing /
21 file_base = "/usr/portage/distfiles"
22 torrent_base = "/tmp/torrents"
23
24 COMMENT = "Gentoo Linux BitTorrent Mirror System"
25 tracker = "http://egret.gentoo.org:6969/announce"
26
27 # Useless variables
28 cols = 80
29
30
31 import BitTorrent
32 from BitTorrent.download import download
33 from threading import Event,Thread
34 import types
35 import re
36 from os import listdir, getcwd, stat
37 from stat import ST_SIZE
38 from os.path import abspath,isdir,isfile,normpath
39 from sys import argv, stdout, path
40 from cStringIO import StringIO
41 from time import time,ctime,sleep
42
43 from BitTorrent.makemetafile import make_meta_file
44
45 def hours(n):
46 if n == -1:
47 return '<unknown>'
48 if n == 0:
49 return 'complete!'
50 n = long(n)
51 h, r = divmod(n, 60 * 60)
52 m, sec = divmod(r, 60)
53 if h > 1000000:
54 return '<unknown>'
55 if h > 0:
56 return '%d hour %02d min %02d sec' % (h, m, sec)
57 else:
58 return '%d min %02d sec' % (m, sec)
59
60 class HeadlessDisplayer:
61 def __init__(self):
62 self.done = False
63 self.failed = False
64 self.file = ''
65 self.activity = ''
66 self.percentDone = 0.0
67 self.timeEst = 0
68 self.downloadTo = ''
69 self.downRate = 0.0
70 self.upRate = 0.0
71 self.downTotal = 0.0
72 self.upTotal = 0
73 self.errors = []
74 self.last_update_time = 0
75
76 def finished(self):
77 self.done = True
78 self.percentDone = 100
79 self.timeEst = 0
80 self.downRate = 0
81 self.display({})
82
83 def failed(self):
84 self.done = True
85 self.failed = True
86 self.percentDone = 0
87 self.timeEst = -1
88 self.downRate = 0
89 self.display({})
90
91 def error(self, errormsg):
92 self.errors.append(errormsg)
93 self.display({})
94
95 def display(self, dict):
96 if ((self.last_update_time + 1) > time()):
97 if dict.get('fractionDone') not in (0.0, 1.0):
98 if not dict.has_key('activity'):
99 return
100 self.last_update_time = time()
101 if dict.has_key('fractionDone'):
102 self.percentDone = float(int(dict['fractionDone'] * 1000)) / 10
103 if dict.has_key('timeEst'):
104 self.timeEst = long(dict['timeEst'])
105 if dict.has_key('activity') and not self.done:
106 self.activity = dict['activity']
107 if dict.has_key('downRate'):
108 self.downRate = float(dict['downRate']) / (1 << 10)
109 if dict.has_key('upRate'):
110 self.upRate = float(dict['upRate']) / (1 << 10)
111 if dict.has_key('upTotal'):
112 self.upTotal = dict['upTotal']
113 if dict.has_key('downTotal'):
114 self.downTotal = dict['downTotal']
115 for x in self.errors:
116 print self.file+": "+str(x)
117 self.errors = []
118 return
119
120 def chooseFile(self, default, size, saveas, dir):
121 self.file = '%s (%.1f MB)' % (default, float(size) / (1 << 20))
122 if saveas != '':
123 default = saveas
124 self.downloadTo = abspath(default)
125 return default
126
127 def newpath(self, path):
128 self.downloadTo = path
129
130 def prefix_array(array,prefix,doblanks=1):
131 """Prepends a given prefix to each element in an Array/List/Tuple.
132 Returns a List."""
133 if type(array) not in [types.ListType, types.TupleType]:
134 raise TypeError, "List or Tuple expected. Got %s" % type(array)
135 newarray=[]
136 for x in array:
137 if x or doblanks:
138 newarray.append(prefix + x)
139 else:
140 newarray.append(x)
141 return newarray
142
143
144 def list_files(dirlist):
145 file_list = []
146 for mydir in dirlist:
147 if isdir(mydir):
148 try:
149 file_list += list_files(prefix_array(listdir(mydir),mydir+"/"))
150 except Exception,e:
151 print e
152 elif isfile(mydir):
153 file_list.append(mydir)
154 else:
155 print "Unknown type:",mydir
156 return file_list
157
158
159 def progress(amount):
160 # Dummy function for the create-torrent progress meter
161 pass
162
163
164 def writemsg(msg,fileName,overwrite=0, create=0):
165 if overwrite or create:
166 fileHandle = open(fileName, "w")
167 else:
168 fileHandle = open(fileName, "a")
169
170 fileHandle.write(msg+"\n")
171 fileHandle.flush()
172 fileHandle.close()
173 print msg
174
175 if __name__ == '__main__':
176
177 threads = {}
178 dirs = [torrent_base[:],file_base[:]]
179 re_file = re.compile("("+torrent_base[:]+"|"+file_base[:]+")(/.+?)(\.torrent)?$")
180 absmax_upRate = 0.0
181 absmax_upFile = ''
182
183 loop_val = 0
184 while(True):
185 loop_val = (loop_val + 1) % LOOP_MOD
186 files = list_files(dirs)
187 files.sort()
188 new_files = []
189 torrents = []
190
191 for f in files:
192 match = re_file.match(f)
193 if not match:
194 print "Failed to match:",f
195 else:
196 if match.group(1) == torrent_base:
197 if match.group(3): # It's a torrent, yay!
198 if match.group(2) not in torrents and stat(f)[ST_SIZE]>0:
199 torrents.append(match.group(2))
200 while(match.group(2) in new_files):
201 del new_files[new_files.index(match.group(2))]
202 elif torrent_base != file_base and debug:
203 print "Not a torrent file:",f
204 if match.group(1) == file_base:
205 if match.group(3) and torrent_base != file_base and debug:
206 print "Torrent in file directory:",f
207 elif match.group(2) not in torrents:
208 if debug:
209 print "New file:",f
210 new_files.append(match.group(2))
211
212 for t in threads.keys():
213 # Remove threads that we don't have a bittorrent for.
214 if t not in torrents or not os.path.exists(file_base+t):
215 print "Stopping torrent [%s]: %s" % (ctime(),t)
216 threads[t][0]._Thread__stop()
217 del threads[t]
218 while(t in torrents):
219 del torrents[torrents.index(t)]
220
221 elif not threads[t][0].isAlive():
222 print "Thread died [%s]: %s" % (ctime(),t)
223 del threads[t]
224
225 fetching = 0
226 completed = 0
227 stalled = 0
228 totalup = 0
229 rateup = 0
230 ratedown = 0
231
232 max_up_rate = 0
233 max_up_file = ''
234 max_down_rate = 0
235 max_down_file = ''
236 for t in threads.keys():
237 # Let's do some stats.
238 if not threads[t][1].failed and threads[t][0].isAlive():
239 if threads[t][1].done:
240 completed += 1
241 else:
242 fetching += 1
243 if threads[t][1].downRate == 0:
244 stalled += 1
245
246 totalup += threads[t][1].upTotal
247
248 rateup += threads[t][1].upRate
249 if threads[t][1].upRate > max_up_rate:
250 max_up_rate = threads[t][1].upRate
251 max_up_file = threads[t][1].file[:]
252 if threads[t][1].upRate > absmax_upRate:
253 absmax_upRate = threads[t][1].upRate
254 absmax_upFile = threads[t][1].file[:]
255
256 ratedown += threads[t][1].downRate
257 if threads[t][1].downRate > max_down_rate:
258 max_down_rate = threads[t][1].downRate
259 max_down_file = threads[t][1].file[:]
260
261 if os.path.exists(".stats"):
262 if ROTATE_LOGS:
263 filename = "bt_daemon-%d-%04d.stats" % (os.getpid(),loop_val)
264 makenew = 1
265 else:
266 filename = "bt_daemon-%d.stats" % (os.getpid())
267 makenew = 0
268 writemsg("",filename,overwrite=makenew)
269 writemsg("Stats: %s" % (ctime()) ,filename)
270 writemsg("Seeding: %d" % (completed) ,filename)
271 writemsg("Fetching: %d(%d)" % (fetching,stalled) ,filename)
272 writemsg("Total up: %.3f MB" % (totalup) ,filename)
273 writemsg("UpRate: %.2f k/s" % (rateup) ,filename)
274 writemsg("UpMaxNow: %.2f k/s '%s'" % (max_up_rate,max_up_file) ,filename)
275 writemsg("UpAbsMax: %.2f k/s '%s'" % (absmax_upRate,absmax_upFile) ,filename)
276 writemsg("DownRate: %.2f k/s" % (ratedown) ,filename)
277 writemsg("DownBest: %.2f k/s '%s'" % (max_down_rate,max_down_file) ,filename)
278 writemsg("",filename)
279
280 for t in torrents:
281 # Start threads up if we have load free for them. Startup is harsh.
282 if t not in threads.keys() and ((len(threads.keys())-completed) < MAX_STARTING_THREADS):
283 print "Starting torrent [%s]: %s" % (ctime(),t)
284 h = HeadlessDisplayer()
285 try:
286 if not os.path.isdir(file_base+"/"+os.path.dirname(t)):
287 os.makedirs(file_base+"/"+os.path.dirname(t))
288 download_args = [
289 # responsefile -> Torrent filename
290 '--responsefile', torrent_base+"/"+t+".torrent",
291 # saveas -> Real filename
292 '--saveas', file_base+"/"+t,
293 # max_slice_length -> How much we can get in a single request
294 '--max_slice_length', "65536",
295 # timeout_check_interval -> Time between timeout checks
296 '--timeout_check_interval', '30',
297 # timeout -> How long until we time out a connection
298 '--timeout', '120',
299 # minport -> Lowest port allowed for files
300 '--minport', '6881',
301 # maxport -> Highest port allowed for files
302 '--maxport', '7881',
303 # request_backlog -> How many requests per pipe do we hold
304 '--request_backlog', '20',
305 # download_slice_size -> How much do we request?
306 '--download_slice_size', '65536',
307 # max_uploads -> How many do we serve out?
308 '--max_uploads', '20',
309 # check_hashes -> Do we check the on-disk files before joining?
310 '--check_hashes', '0'
311 ]
312 t_thread = Thread(target=download,name="BitTorrent::"+t,args=(download_args, h.chooseFile, h.display, h.finished, h.error, Event(), cols))
313 threads[t] = [t_thread,h]
314 t_thread.start()
315 sleep(spawn_thread_delay)
316 except Exception, e:
317 print "Failed to start thread: %s" % (t)
318 print "Reason: %s" % (str(e))
319
320 for f in new_files:
321 # Create the torrents for files that don't have them.
322 try:
323 print "Creating torrent [%s]: %s" % (ctime(),f)
324 if not os.path.isdir(torrent_base+"/"+os.path.dirname(f)):
325 os.makedirs(torrent_base+"/"+os.path.dirname(f))
326 make_meta_file(file_base+"/"+f,tracker,piece_size_pow2,progress=progress,comment=COMMENT,target=torrent_base+"/"+f+".torrent")
327 except Exception, e:
328 print "Failed to create torrent: "+str(e)
329
330 sleep(cycle_sleep)

  ViewVC Help
Powered by ViewVC 1.1.20