/[gentoo-src]/bittorrent/bt_daemon.py
Gentoo

Contents of /bittorrent/bt_daemon.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.2 - (show annotations) (download) (as text)
Sat May 1 20:28:54 2004 UTC (10 years, 3 months ago) by carpaski
Branch: MAIN
Changes since 1.1: +178 -81 lines
File MIME type: text/x-python
New modifications... stats lots of little fun things.

1 #!/usr/bin/python -O
2
3 # Greatly modified by carpaski@gentoo.org
4 # btdownloadheadless written by Bram Cohen
5 # see LICENSE.txt for license information
6
7 import os
8 os.nice(5)
9
10 # This is 2^x ... put the x below -- Chunk size for sha1 -- 18 == 256k
11 piece_size_pow2 = 18
12 MAX_STARTING_THREADS = 15
13 LOOP_MOD = 4320 # Rotating value (for logs)
14 ROTATE_LOGS = 0 # Make one log or roate into loop_mod logs
15 cycle_sleep = 20
16 spawn_thread_delay = 0
17 debug = 0
18
19 # No trailing /
20 file_base = "/usr/portage/distfiles"
21 torrent_base = "/tmp/torrents"
22
23 COMMENT = "Gentoo Linux BitTorrent Mirror System"
24 tracker = "http://egret.gentoo.org:6969/announce"
25
26 # Useless variables
27 cols = 80
28
29
30 import BitTorrent
31 from BitTorrent.download import download
32 from threading import Event,Thread
33 import types
34 import re
35 from os import listdir, getcwd, stat
36 from stat import ST_SIZE
37 from os.path import abspath,isdir,isfile,normpath
38 from sys import argv, stdout, path
39 from cStringIO import StringIO
40 from time import time,ctime,sleep
41
42 from BitTorrent.makemetafile import make_meta_file
43
44 def hours(n):
45 if n == -1:
46 return '<unknown>'
47 if n == 0:
48 return 'complete!'
49 n = long(n)
50 h, r = divmod(n, 60 * 60)
51 m, sec = divmod(r, 60)
52 if h > 1000000:
53 return '<unknown>'
54 if h > 0:
55 return '%d hour %02d min %02d sec' % (h, m, sec)
56 else:
57 return '%d min %02d sec' % (m, sec)
58
59 class HeadlessDisplayer:
60 def __init__(self):
61 self.done = False
62 self.failed = False
63 self.file = ''
64 self.activity = ''
65 self.percentDone = 0.0
66 self.timeEst = 0
67 self.downloadTo = ''
68 self.downRate = 0.0
69 self.upRate = 0.0
70 self.downTotal = 0.0
71 self.upTotal = 0
72 self.errors = []
73 self.last_update_time = 0
74
75 def finished(self):
76 self.done = True
77 self.percentDone = 100
78 self.timeEst = 0
79 self.downRate = 0
80 self.display({})
81
82 def failed(self):
83 self.done = True
84 self.failed = True
85 self.percentDone = 0
86 self.timeEst = -1
87 self.downRate = 0
88 self.display({})
89
90 def error(self, errormsg):
91 self.errors.append(errormsg)
92 self.display({})
93
94 def display(self, dict):
95 if ((self.last_update_time + 1) > time()):
96 if dict.get('fractionDone') not in (0.0, 1.0):
97 if not dict.has_key('activity'):
98 return
99 self.last_update_time = time()
100 if dict.has_key('fractionDone'):
101 self.percentDone = float(int(dict['fractionDone'] * 1000)) / 10
102 if dict.has_key('timeEst'):
103 self.timeEst = long(dict['timeEst'])
104 if dict.has_key('activity') and not self.done:
105 self.activity = dict['activity']
106 if dict.has_key('downRate'):
107 self.downRate = float(dict['downRate']) / (1 << 10)
108 if dict.has_key('upRate'):
109 self.upRate = float(dict['upRate']) / (1 << 10)
110 if dict.has_key('upTotal'):
111 self.upTotal = dict['upTotal']
112 if dict.has_key('downTotal'):
113 self.downTotal = dict['downTotal']
114 for x in self.errors:
115 print self.file+": "+str(x)
116 self.errors = []
117 return
118
119 def chooseFile(self, default, size, saveas, dir):
120 self.file = '%s (%.1f MB)' % (default, float(size) / (1 << 20))
121 if saveas != '':
122 default = saveas
123 self.downloadTo = abspath(default)
124 return default
125
126 def newpath(self, path):
127 self.downloadTo = path
128
129 def prefix_array(array,prefix,doblanks=1):
130 """Prepends a given prefix to each element in an Array/List/Tuple.
131 Returns a List."""
132 if type(array) not in [types.ListType, types.TupleType]:
133 raise TypeError, "List or Tuple expected. Got %s" % type(array)
134 newarray=[]
135 for x in array:
136 if x or doblanks:
137 newarray.append(prefix + x)
138 else:
139 newarray.append(x)
140 return newarray
141
142
143 def list_files(dirlist):
144 file_list = []
145 for mydir in dirlist:
146 if isdir(mydir):
147 try:
148 file_list += list_files(prefix_array(listdir(mydir),mydir+"/"))
149 except Exception,e:
150 print e
151 elif isfile(mydir):
152 file_list.append(mydir)
153 else:
154 print "Unknown type:",mydir
155 return file_list
156
157
158 def progress(amount):
159 # Dummy function for the create-torrent progress meter
160 pass
161
162
163 def writemsg(msg,fileName,overwrite=0, create=0):
164 if overwrite or create:
165 fileHandle = open(fileName, "w")
166 else:
167 fileHandle = open(fileName, "a")
168
169 fileHandle.write(msg+"\n")
170 fileHandle.flush()
171 fileHandle.close()
172 print msg
173
174 if __name__ == '__main__':
175
176 threads = {}
177 dirs = [torrent_base[:],file_base[:]]
178 re_file = re.compile("("+torrent_base[:]+"|"+file_base[:]+")(/.+?)(\.torrent)?$")
179 absmax_upRate = 0.0
180 absmax_upFile = ''
181
182 loop_val = 0
183 while(True):
184 loop_val = (loop_val + 1) % LOOP_MOD
185 files = list_files(dirs)
186 files.sort()
187 new_files = []
188 torrents = []
189
190 for f in files:
191 match = re_file.match(f)
192 if not match:
193 print "Failed to match:",f
194 else:
195 if match.group(1) == torrent_base:
196 if match.group(3): # It's a torrent, yay!
197 if match.group(2) not in torrents and stat(f)[ST_SIZE]>0:
198 torrents.append(match.group(2))
199 while(match.group(2) in new_files):
200 del new_files[new_files.index(match.group(2))]
201 elif torrent_base != file_base and debug:
202 print "Not a torrent file:",f
203 if match.group(1) == file_base:
204 if match.group(3) and torrent_base != file_base and debug:
205 print "Torrent in file directory:",f
206 elif match.group(2) not in torrents:
207 if debug:
208 print "New file:",f
209 new_files.append(match.group(2))
210
211 for t in threads.keys():
212 # Remove threads that we don't have a bittorrent for.
213 if t not in torrents or not os.path.exists(file_base+t):
214 print "Stopping torrent [%s]: %s" % (ctime(),t)
215 threads[t][0]._Thread__stop()
216 del threads[t]
217 while(t in torrents):
218 del torrents[torrents.index(t)]
219
220 elif not threads[t][0].isAlive():
221 print "Thread died [%s]: %s" % (ctime(),t)
222 del threads[t]
223
224 fetching = 0
225 completed = 0
226 stalled = 0
227 totalup = 0
228 rateup = 0
229 ratedown = 0
230
231 max_up_rate = 0
232 max_up_file = ''
233 max_down_rate = 0
234 max_down_file = ''
235 for t in threads.keys():
236 # Let's do some stats.
237 if not threads[t][1].failed and threads[t][0].isAlive():
238 if threads[t][1].done:
239 completed += 1
240 else:
241 fetching += 1
242 if threads[t][1].downRate == 0:
243 stalled += 1
244
245 totalup += threads[t][1].upTotal
246
247 rateup += threads[t][1].upRate
248 if threads[t][1].upRate > max_up_rate:
249 max_up_rate = threads[t][1].upRate
250 max_up_file = threads[t][1].file[:]
251 if threads[t][1].upRate > absmax_upRate:
252 absmax_upRate = threads[t][1].upRate
253 absmax_upFile = threads[t][1].file[:]
254
255 ratedown += threads[t][1].downRate
256 if threads[t][1].downRate > max_down_rate:
257 max_down_rate = threads[t][1].downRate
258 max_down_file = threads[t][1].file[:]
259
260 if os.path.exists(".stats"):
261 if ROTATE_LOGS:
262 filename = "bt_daemon-%d-%4d.stats" % (os.getpid(),loop_val)
263 makenew = 1
264 else:
265 filename = "bt_daemon-%d.stats" % (os.getpid())
266 makenew = 0
267 writemsg("",filename,overwrite=makenew)
268 writemsg("Stats: %s" % (ctime()) ,filename)
269 writemsg("Seeding: %d" % (completed) ,filename)
270 writemsg("Fetching: %d(%d)" % (fetching,stalled) ,filename)
271 writemsg("Total up: %.3f MB" % (totalup) ,filename)
272 writemsg("UpRate: %.2f k/s" % (rateup) ,filename)
273 writemsg("UpMaxNow: %.2f k/s '%s'" % (max_up_rate,max_up_file) ,filename)
274 writemsg("UpAbsMax: %.2f k/s" % (absmax_upRate) ,filename)
275 writemsg("DownRate: %.2f k/s" % (ratedown) ,filename)
276 writemsg("DownBest: %.2f k/s '%s'" % (max_down_rate,max_down_file) ,filename)
277 writemsg("",filename)
278
279 for t in torrents:
280 # Start threads up if we have load free for them. Startup is harsh.
281 if t not in threads.keys() and ((len(threads.keys())-completed) < MAX_STARTING_THREADS):
282 print "Starting torrent [%s]: %s" % (ctime(),t)
283 h = HeadlessDisplayer()
284 try:
285 if not os.path.isdir(file_base+"/"+os.path.dirname(t)):
286 os.makedirs(file_base+"/"+os.path.dirname(t))
287 t_thread = Thread(target=download,name="BitTorrent::"+t,args=(['--responsefile', torrent_base+"/"+t+".torrent", '--saveas', file_base+"/"+t], h.chooseFile, h.display, h.finished, h.error, Event(), cols))
288 threads[t] = [t_thread,h]
289 t_thread.start()
290 sleep(spawn_thread_delay)
291 except Exception, e:
292 print "Failed to start thread: %s" % (t)
293 print "Reason: %s" % (str(e))
294
295 for f in new_files:
296 # Create the torrents for files that don't have them.
297 try:
298 print "Creating torrent [%s]: %s" % (ctime(),f)
299 if not os.path.isdir(torrent_base+"/"+os.path.dirname(f)):
300 os.makedirs(torrent_base+"/"+os.path.dirname(f))
301 make_meta_file(file_base+"/"+f,tracker,piece_size_pow2,progress=progress,comment=COMMENT,target=torrent_base+"/"+f+".torrent")
302 except Exception, e:
303 print "Failed to create torrent: "+str(e)
304
305 sleep(cycle_sleep)

  ViewVC Help
Powered by ViewVC 1.1.20