/[gentoo-src]/bittorrent/bt_daemon.py
Gentoo

Contents of /bittorrent/bt_daemon.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.2 - (hide annotations) (download) (as text)
Sat May 1 20:28:54 2004 UTC (10 years, 4 months ago) by carpaski
Branch: MAIN
Changes since 1.1: +178 -81 lines
File MIME type: text/x-python
New modifications... stats lots of little fun things.

1 carpaski 1.1 #!/usr/bin/python -O
2    
3     # Greatly modified by carpaski@gentoo.org
4     # btdownloadheadless written by Bram Cohen
5     # see LICENSE.txt for license information
6    
7 carpaski 1.2 import os
8     os.nice(5)
9 carpaski 1.1
10     # This is 2^x ... put the x below -- Chunk size for sha1 -- 18 == 256k
11 carpaski 1.2 piece_size_pow2 = 18
12     MAX_STARTING_THREADS = 15
13     LOOP_MOD = 4320 # Rotating value (for logs)
14     ROTATE_LOGS = 0 # Make one log or roate into loop_mod logs
15     cycle_sleep = 20
16     spawn_thread_delay = 0
17     debug = 0
18 carpaski 1.1
19     # No trailing /
20     file_base = "/usr/portage/distfiles"
21     torrent_base = "/tmp/torrents"
22    
23 carpaski 1.2 COMMENT = "Gentoo Linux BitTorrent Mirror System"
24     tracker = "http://egret.gentoo.org:6969/announce"
25 carpaski 1.1
26     # Useless variables
27     cols = 80
28 carpaski 1.2
29    
30 carpaski 1.1 import BitTorrent
31     from BitTorrent.download import download
32     from threading import Event,Thread
33     import types
34     import re
35     from os import listdir, getcwd, stat
36     from stat import ST_SIZE
37     from os.path import abspath,isdir,isfile,normpath
38     from sys import argv, stdout, path
39     from cStringIO import StringIO
40     from time import time,ctime,sleep
41    
42     from BitTorrent.makemetafile import make_meta_file
43    
44     def hours(n):
45 carpaski 1.2 if n == -1:
46     return '<unknown>'
47     if n == 0:
48     return 'complete!'
49     n = long(n)
50     h, r = divmod(n, 60 * 60)
51     m, sec = divmod(r, 60)
52     if h > 1000000:
53     return '<unknown>'
54     if h > 0:
55     return '%d hour %02d min %02d sec' % (h, m, sec)
56     else:
57     return '%d min %02d sec' % (m, sec)
58 carpaski 1.1
59     class HeadlessDisplayer:
60 carpaski 1.2 def __init__(self):
61     self.done = False
62     self.failed = False
63     self.file = ''
64     self.activity = ''
65     self.percentDone = 0.0
66     self.timeEst = 0
67     self.downloadTo = ''
68     self.downRate = 0.0
69     self.upRate = 0.0
70     self.downTotal = 0.0
71     self.upTotal = 0
72     self.errors = []
73     self.last_update_time = 0
74    
75     def finished(self):
76     self.done = True
77     self.percentDone = 100
78     self.timeEst = 0
79     self.downRate = 0
80     self.display({})
81    
82     def failed(self):
83     self.done = True
84     self.failed = True
85     self.percentDone = 0
86     self.timeEst = -1
87     self.downRate = 0
88     self.display({})
89    
90     def error(self, errormsg):
91     self.errors.append(errormsg)
92     self.display({})
93    
94     def display(self, dict):
95     if ((self.last_update_time + 1) > time()):
96     if dict.get('fractionDone') not in (0.0, 1.0):
97     if not dict.has_key('activity'):
98     return
99     self.last_update_time = time()
100     if dict.has_key('fractionDone'):
101     self.percentDone = float(int(dict['fractionDone'] * 1000)) / 10
102     if dict.has_key('timeEst'):
103     self.timeEst = long(dict['timeEst'])
104     if dict.has_key('activity') and not self.done:
105     self.activity = dict['activity']
106     if dict.has_key('downRate'):
107     self.downRate = float(dict['downRate']) / (1 << 10)
108     if dict.has_key('upRate'):
109     self.upRate = float(dict['upRate']) / (1 << 10)
110     if dict.has_key('upTotal'):
111     self.upTotal = dict['upTotal']
112     if dict.has_key('downTotal'):
113     self.downTotal = dict['downTotal']
114     for x in self.errors:
115     print self.file+": "+str(x)
116     self.errors = []
117     return
118    
119     def chooseFile(self, default, size, saveas, dir):
120     self.file = '%s (%.1f MB)' % (default, float(size) / (1 << 20))
121     if saveas != '':
122     default = saveas
123     self.downloadTo = abspath(default)
124     return default
125 carpaski 1.1
126 carpaski 1.2 def newpath(self, path):
127     self.downloadTo = path
128 carpaski 1.1
129     def prefix_array(array,prefix,doblanks=1):
130 carpaski 1.2 """Prepends a given prefix to each element in an Array/List/Tuple.
131     Returns a List."""
132     if type(array) not in [types.ListType, types.TupleType]:
133     raise TypeError, "List or Tuple expected. Got %s" % type(array)
134     newarray=[]
135     for x in array:
136     if x or doblanks:
137     newarray.append(prefix + x)
138     else:
139     newarray.append(x)
140     return newarray
141 carpaski 1.1
142    
143     def list_files(dirlist):
144     file_list = []
145     for mydir in dirlist:
146     if isdir(mydir):
147     try:
148     file_list += list_files(prefix_array(listdir(mydir),mydir+"/"))
149     except Exception,e:
150     print e
151     elif isfile(mydir):
152     file_list.append(mydir)
153     else:
154     print "Unknown type:",mydir
155     return file_list
156    
157    
158     def progress(amount):
159     # Dummy function for the create-torrent progress meter
160     pass
161    
162    
163 carpaski 1.2 def writemsg(msg,fileName,overwrite=0, create=0):
164     if overwrite or create:
165     fileHandle = open(fileName, "w")
166     else:
167     fileHandle = open(fileName, "a")
168    
169     fileHandle.write(msg+"\n")
170     fileHandle.flush()
171     fileHandle.close()
172     print msg
173 carpaski 1.1
174     if __name__ == '__main__':
175    
176     threads = {}
177     dirs = [torrent_base[:],file_base[:]]
178     re_file = re.compile("("+torrent_base[:]+"|"+file_base[:]+")(/.+?)(\.torrent)?$")
179 carpaski 1.2 absmax_upRate = 0.0
180     absmax_upFile = ''
181 carpaski 1.1
182 carpaski 1.2 loop_val = 0
183 carpaski 1.1 while(True):
184 carpaski 1.2 loop_val = (loop_val + 1) % LOOP_MOD
185 carpaski 1.1 files = list_files(dirs)
186     files.sort()
187     new_files = []
188     torrents = []
189    
190     for f in files:
191     match = re_file.match(f)
192     if not match:
193     print "Failed to match:",f
194     else:
195     if match.group(1) == torrent_base:
196     if match.group(3): # It's a torrent, yay!
197     if match.group(2) not in torrents and stat(f)[ST_SIZE]>0:
198     torrents.append(match.group(2))
199     while(match.group(2) in new_files):
200     del new_files[new_files.index(match.group(2))]
201     elif torrent_base != file_base and debug:
202     print "Not a torrent file:",f
203     if match.group(1) == file_base:
204     if match.group(3) and torrent_base != file_base and debug:
205     print "Torrent in file directory:",f
206     elif match.group(2) not in torrents:
207     if debug:
208     print "New file:",f
209     new_files.append(match.group(2))
210    
211     for t in threads.keys():
212     # Remove threads that we don't have a bittorrent for.
213 carpaski 1.2 if t not in torrents or not os.path.exists(file_base+t):
214 carpaski 1.1 print "Stopping torrent [%s]: %s" % (ctime(),t)
215     threads[t][0]._Thread__stop()
216     del threads[t]
217     while(t in torrents):
218     del torrents[torrents.index(t)]
219    
220     elif not threads[t][0].isAlive():
221     print "Thread died [%s]: %s" % (ctime(),t)
222     del threads[t]
223 carpaski 1.2
224     fetching = 0
225     completed = 0
226     stalled = 0
227     totalup = 0
228     rateup = 0
229     ratedown = 0
230    
231     max_up_rate = 0
232     max_up_file = ''
233     max_down_rate = 0
234     max_down_file = ''
235     for t in threads.keys():
236     # Let's do some stats.
237     if not threads[t][1].failed and threads[t][0].isAlive():
238     if threads[t][1].done:
239     completed += 1
240     else:
241     fetching += 1
242     if threads[t][1].downRate == 0:
243     stalled += 1
244    
245     totalup += threads[t][1].upTotal
246    
247     rateup += threads[t][1].upRate
248     if threads[t][1].upRate > max_up_rate:
249     max_up_rate = threads[t][1].upRate
250     max_up_file = threads[t][1].file[:]
251     if threads[t][1].upRate > absmax_upRate:
252     absmax_upRate = threads[t][1].upRate
253     absmax_upFile = threads[t][1].file[:]
254    
255     ratedown += threads[t][1].downRate
256     if threads[t][1].downRate > max_down_rate:
257     max_down_rate = threads[t][1].downRate
258     max_down_file = threads[t][1].file[:]
259    
260     if os.path.exists(".stats"):
261     if ROTATE_LOGS:
262     filename = "bt_daemon-%d-%4d.stats" % (os.getpid(),loop_val)
263     makenew = 1
264     else:
265     filename = "bt_daemon-%d.stats" % (os.getpid())
266     makenew = 0
267     writemsg("",filename,overwrite=makenew)
268     writemsg("Stats: %s" % (ctime()) ,filename)
269     writemsg("Seeding: %d" % (completed) ,filename)
270     writemsg("Fetching: %d(%d)" % (fetching,stalled) ,filename)
271     writemsg("Total up: %.3f MB" % (totalup) ,filename)
272     writemsg("UpRate: %.2f k/s" % (rateup) ,filename)
273     writemsg("UpMaxNow: %.2f k/s '%s'" % (max_up_rate,max_up_file) ,filename)
274     writemsg("UpAbsMax: %.2f k/s" % (absmax_upRate) ,filename)
275     writemsg("DownRate: %.2f k/s" % (ratedown) ,filename)
276     writemsg("DownBest: %.2f k/s '%s'" % (max_down_rate,max_down_file) ,filename)
277     writemsg("",filename)
278 carpaski 1.1
279     for t in torrents:
280 carpaski 1.2 # Start threads up if we have load free for them. Startup is harsh.
281     if t not in threads.keys() and ((len(threads.keys())-completed) < MAX_STARTING_THREADS):
282 carpaski 1.1 print "Starting torrent [%s]: %s" % (ctime(),t)
283     h = HeadlessDisplayer()
284     try:
285     if not os.path.isdir(file_base+"/"+os.path.dirname(t)):
286     os.makedirs(file_base+"/"+os.path.dirname(t))
287     t_thread = Thread(target=download,name="BitTorrent::"+t,args=(['--responsefile', torrent_base+"/"+t+".torrent", '--saveas', file_base+"/"+t], h.chooseFile, h.display, h.finished, h.error, Event(), cols))
288     threads[t] = [t_thread,h]
289     t_thread.start()
290 carpaski 1.2 sleep(spawn_thread_delay)
291 carpaski 1.1 except Exception, e:
292     print "Failed to start thread: %s" % (t)
293     print "Reason: %s" % (str(e))
294    
295     for f in new_files:
296     # Create the torrents for files that don't have them.
297     try:
298     print "Creating torrent [%s]: %s" % (ctime(),f)
299     if not os.path.isdir(torrent_base+"/"+os.path.dirname(f)):
300     os.makedirs(torrent_base+"/"+os.path.dirname(f))
301     make_meta_file(file_base+"/"+f,tracker,piece_size_pow2,progress=progress,comment=COMMENT,target=torrent_base+"/"+f+".torrent")
302     except Exception, e:
303     print "Failed to create torrent: "+str(e)
304    
305 carpaski 1.2 sleep(cycle_sleep)

  ViewVC Help
Powered by ViewVC 1.1.20