/[gentoo-src]/bittorrent/bt_daemon.py
Gentoo

Diff of /bittorrent/bt_daemon.py

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

Revision 1.3 Revision 1.4
1#!/usr/bin/python -O 1#!/usr/bin/python -O
2# $Id: bt_daemon.py,v 1.3 2004/05/02 01:41:19 carpaski Exp $ 2# $Id: bt_daemon.py,v 1.4 2004/05/25 23:51:44 carpaski Exp $
3 3#
4# Greatly modified by carpaski@gentoo.org 4# Greatly modified by carpaski@gentoo.org
5# btdownloadheadless written by Bram Cohen 5# btdownloadheadless written by Bram Cohen
6# see LICENSE.txt for license information 6# see LICENSE.txt for license information
7 7
8import os 8import os,sys
9os.nice(5) 9os.nice(5)
10 10
11# This is 2^x ... put the x below -- Chunk size for sha1 -- 18 == 256k 11sys.path = ["/usr/lib/portage/pym"] + sys.path + ["/usr/bin"]
12piece_size_pow2 = 18 12from portage import getconfig
13MAX_STARTING_THREADS = 10
14LOOP_MOD = 4320 # Rotating value (for logs)
15ROTATE_LOGS = 0 # Make one log or roate into loop_mod logs
16cycle_sleep = 20
17spawn_thread_delay = 0
18debug = 0
19
20# No trailing /
21file_base = "/usr/portage/distfiles"
22torrent_base = "/tmp/torrents"
23
24COMMENT = "Gentoo Linux BitTorrent Mirror System"
25tracker = "http://egret.gentoo.org:6969/announce"
26
27# Useless variables
28cols = 80
29
30 13
31import BitTorrent 14import BitTorrent
32from BitTorrent.download import download 15from BitTorrent.download import download
33from threading import Event,Thread 16from threading import Event,Thread
34import types 17import types
35import re 18import re
36from os import listdir, getcwd, stat 19from os import listdir, getcwd, stat
37from stat import ST_SIZE 20from stat import ST_SIZE,ST_MTIME
38from os.path import abspath,isdir,isfile,normpath 21from os.path import abspath,isdir,isfile,normpath
39from sys import argv, stdout, path 22from sys import argv, stdout, path
40from cStringIO import StringIO 23from cStringIO import StringIO
41from time import time,ctime,sleep 24from time import time,ctime,sleep
42 25
43from BitTorrent.makemetafile import make_meta_file 26from btmakemetafile import make_meta_file
44 27
45def hours(n): 28def hours(n):
46 if n == -1: 29 if n == -1:
47 return '<unknown>' 30 return '<unknown>'
48 if n == 0: 31 if n == 0:
138 newarray.append(prefix + x) 121 newarray.append(prefix + x)
139 else: 122 else:
140 newarray.append(x) 123 newarray.append(x)
141 return newarray 124 return newarray
142 125
143
144def list_files(dirlist): 126def list_files(dirlist):
145 file_list = [] 127 file_list = []
146 for mydir in dirlist: 128 for mydir in dirlist:
147 if isdir(mydir): 129 if isdir(mydir):
148 try: 130 try:
170 fileHandle.write(msg+"\n") 152 fileHandle.write(msg+"\n")
171 fileHandle.flush() 153 fileHandle.flush()
172 fileHandle.close() 154 fileHandle.close()
173 print msg 155 print msg
174 156
157
158# Useless variables
159cols = 80
160
161# This is 2^x ... put the x below -- Chunk size for sha1 -- 18 == 256k
162piece_size_pow2 = 18
163max_starting_threads = 10
164loop_mod = 4320 # Rotating value (for logs)
165rotate_logs = 0 # Make one log or roate into loop_mod logs
166cycle_sleep = 20
167spawn_thread_delay = 0
168debug = 0
169minimum_age = 30
170port_min = 6881
171port_max = 7280
172check_hashes = 0
173
174# No trailing /
175file_base = "/usr/portage/distfiles"
176torrent_base = "/tmp/torrents"
177
178comment = "Gentoo Linux BitTorrent Mirror System"
179tracker = "http://egret.gentoo.org:6969/announce"
180
175if __name__ == '__main__': 181if __name__ == '__main__':
182 mydict = getconfig("bt_daemon.conf")
183 if not mydict:
184 print "bt_daemon.conf is missing"
185 sys.exit(1)
186
187 if mydict.has_key("file_base") and mydict["file_base"]:
188 file_base = os.path.normpath(mydict["file_base"])
189 else:
190 print "file_base not specified in config."
191 sys.exit(1)
192
193 if mydict.has_key("torrent_base") and mydict["torrent_base"]:
194 torrent_base = os.path.normpath(mydict["torrent_base"])
195 else:
196 print "torrent_base not specified in config."
197 sys.exit(1)
176 198
177 threads = {} 199 threads = {}
178 dirs = [torrent_base[:],file_base[:]] 200 dirs = [torrent_base[:],file_base[:]]
179 re_file = re.compile("("+torrent_base[:]+"|"+file_base[:]+")(/.+?)(\.torrent)?$") 201 re_file = re.compile("("+torrent_base[:]+"|"+file_base[:]+")(/.+?)(\.torrent)?$")
180 absmax_upRate = 0.0 202 absmax_upRate = 0.0
181 absmax_upFile = '' 203 absmax_upFile = ''
182 204
183 loop_val = 0 205 loop_val = 0
184 while(True): 206 while(True):
207 mydict = getconfig("bt_daemon.conf")
208 if not mydict:
209 print "bt_daemon.conf is missing"
210 sys.exit(1)
211
212 if mydict.has_key("piece_size_pow2"):
213 piece_size_pow2 = int(mydict["piece_size_pow2"])
214 if mydict.has_key("max_starting_threads"):
215 max_starting_threads = int(mydict["max_starting_threads"])
216 if mydict.has_key("loop_mod"):
217 loop_mod = int(mydict["loop_mod"])
218 if mydict.has_key("rotate_logs"):
219 rotate_logs = int(mydict["rotate_logs"])
220 if mydict.has_key("cycle_sleep"):
221 cycle_sleep = int(mydict["cycle_sleep"])
222 if mydict.has_key("spawn_thread_delay"):
223 spawn_thread_delay = int(mydict["spawn_thread_delay"])
224 if mydict.has_key("debug"):
225 debug = int(mydict["debug"])
226
227 if mydict.has_key("comment"):
228 comment = mydict["comment"]
229 if mydict.has_key("port_min"):
230 port_min = mydict["port_min"]
231 if mydict.has_key("port_max"):
232 port_max = mydict["port_max"]
233 if mydict.has_key("check_hashes"):
234 check_hashes = mydict["check_hashes"]
235
236 if mydict.has_key("tracker") and mydict["tracker"]:
237 tracker = mydict["tracker"]
238 else:
239 print "tracker not specified in config."
240 sys.exit(1)
241
242
185 loop_val = (loop_val + 1) % LOOP_MOD 243 loop_val = (loop_val + 1) % loop_mod
186 files = list_files(dirs) 244 files = list_files(dirs)
187 files.sort() 245 files.sort()
188 new_files = [] 246 new_files = []
189 torrents = [] 247 torrents = []
190 248
193 if not match: 251 if not match:
194 print "Failed to match:",f 252 print "Failed to match:",f
195 else: 253 else:
196 if match.group(1) == torrent_base: 254 if match.group(1) == torrent_base:
197 if match.group(3): # It's a torrent, yay! 255 if match.group(3): # It's a torrent, yay!
198 if match.group(2) not in torrents and stat(f)[ST_SIZE]>0: 256 if match.group(2) not in torrents:
257 mystat = stat(f)
258 # only add if it has a size and is not actively modified.
259 if mystat[ST_SIZE]>0 and (time()-mystat[ST_MTIME])>minimum_age:
199 torrents.append(match.group(2)) 260 torrents.append(match.group(2))
200 while(match.group(2) in new_files): 261 while(match.group(2) in new_files):
201 del new_files[new_files.index(match.group(2))] 262 del new_files[new_files.index(match.group(2))]
202 elif torrent_base != file_base and debug: 263 elif torrent_base != file_base and debug:
203 print "Not a torrent file:",f 264 print "Not a torrent file:",f
204 if match.group(1) == file_base: 265 if match.group(1) == file_base:
205 if match.group(3) and torrent_base != file_base and debug: 266 if match.group(3) and torrent_base != file_base and debug:
206 print "Torrent in file directory:",f 267 print "Torrent in file directory:",f
207 elif match.group(2) not in torrents: 268 elif match.group(2) not in torrents:
208 if debug: 269 if debug:
209 print "New file:",f 270 print "New file:",f
271 mystat = stat(f)
272 if mystat[ST_SIZE]>0 and (time()-mystat[ST_MTIME])>minimum_age:
210 new_files.append(match.group(2)) 273 new_files.append(match.group(2))
211 274
212 for t in threads.keys(): 275 for t in threads.keys():
213 # Remove threads that we don't have a bittorrent for. 276 # Remove threads that we don't have a bittorrent for.
214 if t not in torrents or not os.path.exists(file_base+t): 277 if t not in torrents or not os.path.exists(file_base+t):
215 print "Stopping torrent [%s]: %s" % (ctime(),t) 278 print "Stopping torrent [%s]: %s" % (ctime(),t)
277 writemsg("DownBest: %.2f k/s '%s'" % (max_down_rate,max_down_file) ,filename) 340 writemsg("DownBest: %.2f k/s '%s'" % (max_down_rate,max_down_file) ,filename)
278 writemsg("",filename) 341 writemsg("",filename)
279 342
280 for t in torrents: 343 for t in torrents:
281 # Start threads up if we have load free for them. Startup is harsh. 344 # Start threads up if we have load free for them. Startup is harsh.
282 if t not in threads.keys() and ((len(threads.keys())-completed) < MAX_STARTING_THREADS): 345 if t not in threads.keys() and ((len(threads.keys())-completed) < max_starting_threads):
283 print "Starting torrent [%s]: %s" % (ctime(),t) 346 print "Starting torrent [%s]: %s" % (ctime(),t)
284 h = HeadlessDisplayer() 347 h = HeadlessDisplayer()
285 try: 348 try:
286 if not os.path.isdir(file_base+"/"+os.path.dirname(t)): 349 if not os.path.isdir(file_base+"/"+os.path.dirname(t)):
287 os.makedirs(file_base+"/"+os.path.dirname(t)) 350 os.makedirs(file_base+"/"+os.path.dirname(t))
288 download_args = [ 351 download_args = [
289 # responsefile -> Torrent filename 352 # responsefile -> Torrent filename
290 '--responsefile', torrent_base+"/"+t+".torrent", 353 '--responsefile', torrent_base+"/"+t+".torrent",
291 # saveas -> Real filename 354 # saveas -> Real filename
292 '--saveas', file_base+"/"+t, 355 '--saveas', file_base+"/"+t,
293 # max_slice_length -> How much we can get in a single request 356 # max_slice_length -> How much we can get in a single request
294 '--max_slice_length', "65536", 357 '--max_slice_length', "65536",
295 # timeout_check_interval -> Time between timeout checks 358 # timeout_check_interval -> Time between timeout checks
296 '--timeout_check_interval', '30', 359 '--timeout_check_interval', '30',
297 # timeout -> How long until we time out a connection 360 # timeout -> How long until we time out a connection
298 '--timeout', '120', 361 '--timeout', '120',
299 # minport -> Lowest port allowed for files 362 # minport -> Lowest port allowed for files
300 '--minport', '6881', 363 '--minport', str(port_min),
301 # maxport -> Highest port allowed for files 364 # maxport -> Highest port allowed for files
302 '--maxport', '7881', 365 '--maxport', str(port_max),
303 # request_backlog -> How many requests per pipe do we hold 366 # request_backlog -> How many requests per pipe do we hold
304 '--request_backlog', '20', 367 '--request_backlog', '20',
305 # download_slice_size -> How much do we request? 368 # download_slice_size -> How much do we request?
306 '--download_slice_size', '65536', 369 '--download_slice_size', '65536',
307 # max_uploads -> How many do we serve out? 370 # max_uploads -> How many do we serve out?
308 '--max_uploads', '20', 371 '--max_uploads', '20',
309 # check_hashes -> Do we check the on-disk files before joining? 372 # check_hashes -> Do we check the on-disk files before joining?
310 '--check_hashes', '0' 373 '--check_hashes', str(check_hashes)
311 ] 374 ]
312 t_thread = Thread(target=download,name="BitTorrent::"+t,args=(download_args, h.chooseFile, h.display, h.finished, h.error, Event(), cols)) 375 t_thread = Thread(target=download,name="BitTorrent::"+t,args=(download_args, h.chooseFile, h.display, h.finished, h.error, Event(), cols))
313 threads[t] = [t_thread,h] 376 threads[t] = [t_thread,h]
314 t_thread.start() 377 t_thread.start()
315 sleep(spawn_thread_delay) 378 sleep(spawn_thread_delay)
321 # Create the torrents for files that don't have them. 384 # Create the torrents for files that don't have them.
322 try: 385 try:
323 print "Creating torrent [%s]: %s" % (ctime(),f) 386 print "Creating torrent [%s]: %s" % (ctime(),f)
324 if not os.path.isdir(torrent_base+"/"+os.path.dirname(f)): 387 if not os.path.isdir(torrent_base+"/"+os.path.dirname(f)):
325 os.makedirs(torrent_base+"/"+os.path.dirname(f)) 388 os.makedirs(torrent_base+"/"+os.path.dirname(f))
326 make_meta_file(file_base+"/"+f,tracker,piece_size_pow2,progress=progress,comment=COMMENT,target=torrent_base+"/"+f+".torrent") 389 make_meta_file(file_base+"/"+f,tracker,piece_size_pow2,progress=progress,comment=comment,target=torrent_base+"/"+f+".torrent")
327 except Exception, e: 390 except Exception, e:
328 print "Failed to create torrent: "+str(e) 391 print "Failed to create torrent: "+str(e)
329 392
330 sleep(cycle_sleep) 393 sleep(cycle_sleep)

Legend:
Removed from v.1.3  
changed lines
  Added in v.1.4

  ViewVC Help
Powered by ViewVC 1.1.20