/[gentoo-src]/bittorrent/bt_daemon.py
Gentoo

Contents of /bittorrent/bt_daemon.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1.3 - (hide annotations) (download) (as text)
Sun May 2 01:41:19 2004 UTC (9 years, 11 months ago) by carpaski
Branch: MAIN
Changes since 1.2: +29 -4 lines
File MIME type: text/x-python
More fixes. More control on the client processes.

1 carpaski 1.1 #!/usr/bin/python -O
2 carpaski 1.3 # $Id: $
3 carpaski 1.1
4     # Greatly modified by carpaski@gentoo.org
5     # btdownloadheadless written by Bram Cohen
6     # see LICENSE.txt for license information
7    
8 carpaski 1.2 import os
9     os.nice(5)
10 carpaski 1.1
11     # This is 2^x ... put the x below -- Chunk size for sha1 -- 18 == 256k
12 carpaski 1.2 piece_size_pow2 = 18
13 carpaski 1.3 MAX_STARTING_THREADS = 10
14 carpaski 1.2 LOOP_MOD = 4320 # Rotating value (for logs)
15     ROTATE_LOGS = 0 # Make one log or roate into loop_mod logs
16     cycle_sleep = 20
17     spawn_thread_delay = 0
18     debug = 0
19 carpaski 1.1
20     # No trailing /
21     file_base = "/usr/portage/distfiles"
22     torrent_base = "/tmp/torrents"
23    
24 carpaski 1.2 COMMENT = "Gentoo Linux BitTorrent Mirror System"
25     tracker = "http://egret.gentoo.org:6969/announce"
26 carpaski 1.1
27     # Useless variables
28     cols = 80
29 carpaski 1.2
30    
31 carpaski 1.1 import BitTorrent
32     from BitTorrent.download import download
33     from threading import Event,Thread
34     import types
35     import re
36     from os import listdir, getcwd, stat
37     from stat import ST_SIZE
38     from os.path import abspath,isdir,isfile,normpath
39     from sys import argv, stdout, path
40     from cStringIO import StringIO
41     from time import time,ctime,sleep
42    
43     from BitTorrent.makemetafile import make_meta_file
44    
45     def hours(n):
46 carpaski 1.2 if n == -1:
47     return '<unknown>'
48     if n == 0:
49     return 'complete!'
50     n = long(n)
51     h, r = divmod(n, 60 * 60)
52     m, sec = divmod(r, 60)
53     if h > 1000000:
54     return '<unknown>'
55     if h > 0:
56     return '%d hour %02d min %02d sec' % (h, m, sec)
57     else:
58     return '%d min %02d sec' % (m, sec)
59 carpaski 1.1
60     class HeadlessDisplayer:
61 carpaski 1.2 def __init__(self):
62     self.done = False
63     self.failed = False
64     self.file = ''
65     self.activity = ''
66     self.percentDone = 0.0
67     self.timeEst = 0
68     self.downloadTo = ''
69     self.downRate = 0.0
70     self.upRate = 0.0
71     self.downTotal = 0.0
72     self.upTotal = 0
73     self.errors = []
74     self.last_update_time = 0
75    
76     def finished(self):
77     self.done = True
78     self.percentDone = 100
79     self.timeEst = 0
80     self.downRate = 0
81     self.display({})
82    
83     def failed(self):
84     self.done = True
85     self.failed = True
86     self.percentDone = 0
87     self.timeEst = -1
88     self.downRate = 0
89     self.display({})
90    
91     def error(self, errormsg):
92     self.errors.append(errormsg)
93     self.display({})
94    
95     def display(self, dict):
96     if ((self.last_update_time + 1) > time()):
97     if dict.get('fractionDone') not in (0.0, 1.0):
98     if not dict.has_key('activity'):
99     return
100     self.last_update_time = time()
101     if dict.has_key('fractionDone'):
102     self.percentDone = float(int(dict['fractionDone'] * 1000)) / 10
103     if dict.has_key('timeEst'):
104     self.timeEst = long(dict['timeEst'])
105     if dict.has_key('activity') and not self.done:
106     self.activity = dict['activity']
107     if dict.has_key('downRate'):
108     self.downRate = float(dict['downRate']) / (1 << 10)
109     if dict.has_key('upRate'):
110     self.upRate = float(dict['upRate']) / (1 << 10)
111     if dict.has_key('upTotal'):
112     self.upTotal = dict['upTotal']
113     if dict.has_key('downTotal'):
114     self.downTotal = dict['downTotal']
115     for x in self.errors:
116     print self.file+": "+str(x)
117     self.errors = []
118     return
119    
120     def chooseFile(self, default, size, saveas, dir):
121     self.file = '%s (%.1f MB)' % (default, float(size) / (1 << 20))
122     if saveas != '':
123     default = saveas
124     self.downloadTo = abspath(default)
125     return default
126 carpaski 1.1
127 carpaski 1.2 def newpath(self, path):
128     self.downloadTo = path
129 carpaski 1.1
130     def prefix_array(array,prefix,doblanks=1):
131 carpaski 1.2 """Prepends a given prefix to each element in an Array/List/Tuple.
132     Returns a List."""
133     if type(array) not in [types.ListType, types.TupleType]:
134     raise TypeError, "List or Tuple expected. Got %s" % type(array)
135     newarray=[]
136     for x in array:
137     if x or doblanks:
138     newarray.append(prefix + x)
139     else:
140     newarray.append(x)
141     return newarray
142 carpaski 1.1
143    
144     def list_files(dirlist):
145     file_list = []
146     for mydir in dirlist:
147     if isdir(mydir):
148     try:
149     file_list += list_files(prefix_array(listdir(mydir),mydir+"/"))
150     except Exception,e:
151     print e
152     elif isfile(mydir):
153     file_list.append(mydir)
154     else:
155     print "Unknown type:",mydir
156     return file_list
157    
158    
159     def progress(amount):
160     # Dummy function for the create-torrent progress meter
161     pass
162    
163    
164 carpaski 1.2 def writemsg(msg,fileName,overwrite=0, create=0):
165     if overwrite or create:
166     fileHandle = open(fileName, "w")
167     else:
168     fileHandle = open(fileName, "a")
169    
170     fileHandle.write(msg+"\n")
171     fileHandle.flush()
172     fileHandle.close()
173     print msg
174 carpaski 1.1
175     if __name__ == '__main__':
176    
177     threads = {}
178     dirs = [torrent_base[:],file_base[:]]
179     re_file = re.compile("("+torrent_base[:]+"|"+file_base[:]+")(/.+?)(\.torrent)?$")
180 carpaski 1.2 absmax_upRate = 0.0
181     absmax_upFile = ''
182 carpaski 1.1
183 carpaski 1.2 loop_val = 0
184 carpaski 1.1 while(True):
185 carpaski 1.2 loop_val = (loop_val + 1) % LOOP_MOD
186 carpaski 1.1 files = list_files(dirs)
187     files.sort()
188     new_files = []
189     torrents = []
190    
191     for f in files:
192     match = re_file.match(f)
193     if not match:
194     print "Failed to match:",f
195     else:
196     if match.group(1) == torrent_base:
197     if match.group(3): # It's a torrent, yay!
198     if match.group(2) not in torrents and stat(f)[ST_SIZE]>0:
199     torrents.append(match.group(2))
200     while(match.group(2) in new_files):
201     del new_files[new_files.index(match.group(2))]
202     elif torrent_base != file_base and debug:
203     print "Not a torrent file:",f
204     if match.group(1) == file_base:
205     if match.group(3) and torrent_base != file_base and debug:
206     print "Torrent in file directory:",f
207     elif match.group(2) not in torrents:
208     if debug:
209     print "New file:",f
210     new_files.append(match.group(2))
211    
212     for t in threads.keys():
213     # Remove threads that we don't have a bittorrent for.
214 carpaski 1.2 if t not in torrents or not os.path.exists(file_base+t):
215 carpaski 1.1 print "Stopping torrent [%s]: %s" % (ctime(),t)
216     threads[t][0]._Thread__stop()
217     del threads[t]
218     while(t in torrents):
219     del torrents[torrents.index(t)]
220    
221     elif not threads[t][0].isAlive():
222     print "Thread died [%s]: %s" % (ctime(),t)
223     del threads[t]
224 carpaski 1.2
225     fetching = 0
226     completed = 0
227     stalled = 0
228     totalup = 0
229     rateup = 0
230     ratedown = 0
231    
232     max_up_rate = 0
233     max_up_file = ''
234     max_down_rate = 0
235     max_down_file = ''
236     for t in threads.keys():
237     # Let's do some stats.
238     if not threads[t][1].failed and threads[t][0].isAlive():
239     if threads[t][1].done:
240     completed += 1
241     else:
242     fetching += 1
243     if threads[t][1].downRate == 0:
244     stalled += 1
245    
246     totalup += threads[t][1].upTotal
247    
248     rateup += threads[t][1].upRate
249     if threads[t][1].upRate > max_up_rate:
250     max_up_rate = threads[t][1].upRate
251     max_up_file = threads[t][1].file[:]
252     if threads[t][1].upRate > absmax_upRate:
253     absmax_upRate = threads[t][1].upRate
254     absmax_upFile = threads[t][1].file[:]
255    
256     ratedown += threads[t][1].downRate
257     if threads[t][1].downRate > max_down_rate:
258     max_down_rate = threads[t][1].downRate
259     max_down_file = threads[t][1].file[:]
260    
261     if os.path.exists(".stats"):
262     if ROTATE_LOGS:
263 carpaski 1.3 filename = "bt_daemon-%d-%04d.stats" % (os.getpid(),loop_val)
264 carpaski 1.2 makenew = 1
265     else:
266     filename = "bt_daemon-%d.stats" % (os.getpid())
267     makenew = 0
268     writemsg("",filename,overwrite=makenew)
269     writemsg("Stats: %s" % (ctime()) ,filename)
270     writemsg("Seeding: %d" % (completed) ,filename)
271     writemsg("Fetching: %d(%d)" % (fetching,stalled) ,filename)
272     writemsg("Total up: %.3f MB" % (totalup) ,filename)
273     writemsg("UpRate: %.2f k/s" % (rateup) ,filename)
274     writemsg("UpMaxNow: %.2f k/s '%s'" % (max_up_rate,max_up_file) ,filename)
275 carpaski 1.3 writemsg("UpAbsMax: %.2f k/s '%s'" % (absmax_upRate,absmax_upFile) ,filename)
276 carpaski 1.2 writemsg("DownRate: %.2f k/s" % (ratedown) ,filename)
277     writemsg("DownBest: %.2f k/s '%s'" % (max_down_rate,max_down_file) ,filename)
278     writemsg("",filename)
279 carpaski 1.1
280     for t in torrents:
281 carpaski 1.2 # Start threads up if we have load free for them. Startup is harsh.
282     if t not in threads.keys() and ((len(threads.keys())-completed) < MAX_STARTING_THREADS):
283 carpaski 1.1 print "Starting torrent [%s]: %s" % (ctime(),t)
284     h = HeadlessDisplayer()
285     try:
286     if not os.path.isdir(file_base+"/"+os.path.dirname(t)):
287     os.makedirs(file_base+"/"+os.path.dirname(t))
288 carpaski 1.3 download_args = [
289     # responsefile -> Torrent filename
290     '--responsefile', torrent_base+"/"+t+".torrent",
291     # saveas -> Real filename
292     '--saveas', file_base+"/"+t,
293     # max_slice_length -> How much we can get in a single request
294     '--max_slice_length', "65536",
295     # timeout_check_interval -> Time between timeout checks
296     '--timeout_check_interval', '30',
297     # timeout -> How long until we time out a connection
298     '--timeout', '120',
299     # minport -> Lowest port allowed for files
300     '--minport', '6881',
301     # maxport -> Highest port allowed for files
302     '--maxport', '7881',
303     # request_backlog -> How many requests per pipe do we hold
304     '--request_backlog', '20',
305     # download_slice_size -> How much do we request?
306     '--download_slice_size', '65536',
307     # max_uploads -> How many do we serve out?
308     '--max_uploads', '20',
309     # check_hashes -> Do we check the on-disk files before joining?
310     '--check_hashes', '0'
311     ]
312     t_thread = Thread(target=download,name="BitTorrent::"+t,args=(download_args, h.chooseFile, h.display, h.finished, h.error, Event(), cols))
313 carpaski 1.1 threads[t] = [t_thread,h]
314     t_thread.start()
315 carpaski 1.2 sleep(spawn_thread_delay)
316 carpaski 1.1 except Exception, e:
317     print "Failed to start thread: %s" % (t)
318     print "Reason: %s" % (str(e))
319    
320     for f in new_files:
321     # Create the torrents for files that don't have them.
322     try:
323     print "Creating torrent [%s]: %s" % (ctime(),f)
324     if not os.path.isdir(torrent_base+"/"+os.path.dirname(f)):
325     os.makedirs(torrent_base+"/"+os.path.dirname(f))
326     make_meta_file(file_base+"/"+f,tracker,piece_size_pow2,progress=progress,comment=COMMENT,target=torrent_base+"/"+f+".torrent")
327     except Exception, e:
328     print "Failed to create torrent: "+str(e)
329    
330 carpaski 1.2 sleep(cycle_sleep)

  ViewVC Help
Powered by ViewVC 1.1.20