/[gentoo-src]/bittorrent/bt_daemon.py
Gentoo

Diff of /bittorrent/bt_daemon.py

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

Revision 1.5 Revision 1.6
1#!/usr/bin/python -O 1#!/usr/bin/python -O
2# $Id: bt_daemon.py,v 1.5 2004/06/16 05:15:31 carpaski Exp $ 2# $Id: bt_daemon.py,v 1.6 2004/06/16 08:46:48 carpaski Exp $
3# 3#
4# Greatly modified by carpaski@gentoo.org 4# Greatly modified by carpaski@gentoo.org
5# btdownloadheadless written by Bram Cohen 5# btdownloadheadless written by Bram Cohen
6# see LICENSE.txt for license information 6# see LICENSE.txt for license information
7 7
8import os,sys,string 8import os,sys,string,signal
9os.nice(5) 9os.nice(5)
10 10
11sys.path = ["/usr/lib/portage/pym"] + sys.path + ["/usr/bin"] 11sys.path = ["/usr/lib/portage/pym"] + sys.path + ["/usr/bin"]
12from portage import getconfig 12from portage import getconfig
13 13
14import BitTorrent 14import BitTorrent
15from BitTorrent.download import download 15from BitTorrent.download import download
16 16
17from threading import Event,Thread 17from threading import Event,Thread
18import types,traceback,re,random 18import types,re,random
19 19
20from os import listdir, getcwd, stat 20from os import listdir, getcwd, stat
21from stat import ST_SIZE,ST_MTIME 21from stat import ST_SIZE,ST_MTIME
22from os.path import abspath,isdir,isfile,normpath 22from os.path import abspath,isdir,isfile,normpath
23from sys import argv, stdout, path 23from sys import argv, stdout, path
… …
131 return newarray 131 return newarray
132 132
133def list_files(dirlist): 133def list_files(dirlist):
134 file_list = [] 134 file_list = []
135 for mydir in dirlist: 135 for mydir in dirlist:
136 mys = mydir.split("/")
137 if mys[-1][0] == '.':
138 continue
136 if isdir(mydir): 139 if isdir(mydir):
137 try: 140 try:
138 file_list += list_files(prefix_array(listdir(mydir),mydir+"/")) 141 file_list += list_files(prefix_array(listdir(mydir),mydir+"/"))
139 except Exception,e: 142 except Exception,e:
140 print e 143 print e
… …
165 168
166# Useless variables 169# Useless variables
167cols = 80 170cols = 80
168 171
169# This is 2^x ... put the x below -- Chunk size for sha1 -- 18 == 256k 172# This is 2^x ... put the x below -- Chunk size for sha1 -- 18 == 256k
170piece_size_pow2 = 18 173piece_size_pow2 = 18
171max_starting_threads = 1 174max_starting_threads = 1
172loop_mod = 4320 # Rotating value (for logs) 175loop_mod = 4320 # Rotating value (for logs)
173rotate_logs = 0 # Make one log or roate into loop_mod logs 176rotate_logs = 0 # Make one log or roate into loop_mod logs
174cycle_sleep = 20 177cycle_sleep = 20
175spawn_thread_delay = 0 178spawn_thread_delay = 0
176debug = 0 179debug = 0
177minimum_age = 30 180minimum_age = 120
178port_min = 6881 181port_min = 6881
179port_max = 7280 182port_max = 7280
180check_hashes = 0 183check_hashes = 0
181shuffle_files = 1 184shuffle_files = 0
182master_seed = 0 185master_seed = 0
183min_uploads = 20 186min_uploads = 20
184max_uploads = 20 187max_uploads = 20
185download_slice_size = 65536 188download_slice_size = 65536
186max_slice_length = 131072 189max_slice_length = 131072
187request_backlog = 20 190request_backlog = 20
188timeout = 90 191timeout = 90
189timeout_check_interval = 20 192timeout_check_interval = 20
190stats = 1 193stats = 1
194min_file_size = 5000000
195max_fetches = 3
191 196
192# No trailing / 197# No trailing /
193file_base = "/usr/portage/distfiles" 198file_base = "/usr/portage/distfiles"
194torrent_base = "/tmp/torrents" 199torrent_base = "/tmp/torrents"
195 200
… …
217 threads = {} 222 threads = {}
218 dirs = [torrent_base[:],file_base[:]] 223 dirs = [torrent_base[:],file_base[:]]
219 re_file = re.compile("("+torrent_base[:]+"|"+file_base[:]+")(/.+?)(\.torrent)?$") 224 re_file = re.compile("("+torrent_base[:]+"|"+file_base[:]+")(/.+?)(\.torrent)?$")
220 absmax_upRate = 0.0 225 absmax_upRate = 0.0
221 absmax_upFile = '' 226 absmax_upFile = ''
227
228 noticed_newfiles = []
222 229
223 loop_val = 0 230 loop_val = 0
224 while(True): 231 while(True):
225 mydict = getconfig("bt_daemon.conf") 232 mydict = getconfig("bt_daemon.conf")
226 if not mydict: 233 if not mydict:
… …
231 238
232 if mydict.has_key("piece_size_pow2"): 239 if mydict.has_key("piece_size_pow2"):
233 piece_size_pow2 = int(mydict["piece_size_pow2"]) 240 piece_size_pow2 = int(mydict["piece_size_pow2"])
234 if mydict.has_key("max_starting_threads"): 241 if mydict.has_key("max_starting_threads"):
235 max_starting_threads = int(mydict["max_starting_threads"]) 242 max_starting_threads = int(mydict["max_starting_threads"])
243 if mydict.has_key("max_fetches"):
244 max_fetches = int(mydict["max_fetches"])
236 if mydict.has_key("loop_mod"): 245 if mydict.has_key("loop_mod"):
237 loop_mod = int(mydict["loop_mod"]) 246 loop_mod = int(mydict["loop_mod"])
238 if mydict.has_key("rotate_logs"): 247 if mydict.has_key("rotate_logs"):
239 rotate_logs = int(mydict["rotate_logs"]) 248 rotate_logs = int(mydict["rotate_logs"])
240 if mydict.has_key("cycle_sleep"): 249 if mydict.has_key("cycle_sleep"):
241 cycle_sleep = int(mydict["cycle_sleep"]) 250 cycle_sleep = int(mydict["cycle_sleep"])
242 if mydict.has_key("spawn_thread_delay"): 251 if mydict.has_key("spawn_thread_delay"):
243 spawn_thread_delay = int(mydict["spawn_thread_delay"]) 252 spawn_thread_delay = int(mydict["spawn_thread_delay"])
253 if mydict.has_key("shuffle_files"):
254 shuffle_files = int(mydict["shuffle_files"])
244 if mydict.has_key("debug"): 255 if mydict.has_key("debug"):
245 debug = int(mydict["debug"]) 256 debug = int(mydict["debug"])
246 257
247 if mydict.has_key("min_uploads"): 258 if mydict.has_key("min_uploads"):
248 min_uploads = int(mydict["min_uploads"]) 259 min_uploads = int(mydict["min_uploads"])
… …
278 loop_val = (loop_val + 1) % loop_mod 289 loop_val = (loop_val + 1) % loop_mod
279 files = list_files(dirs) 290 files = list_files(dirs)
280 files.sort() 291 files.sort()
281 new_files = [] 292 new_files = []
282 torrents = [] 293 torrents = []
294 changing = []
283 295
284 if shuffle_files: 296 if shuffle_files:
285 random.shuffle(files) 297 random.shuffle(files)
286 298
287 for f in files: 299 for f in files:
… …
295 if match.group(2) not in torrents: 307 if match.group(2) not in torrents:
296 mystat = stat(f) 308 mystat = stat(f)
297 # only add if it has a size and is not actively modified. 309 # only add if it has a size and is not actively modified.
298 if mystat[ST_SIZE]>0 and (time()-mystat[ST_MTIME])>minimum_age: 310 if mystat[ST_SIZE]>0 and (time()-mystat[ST_MTIME])>minimum_age:
299 torrents.append(match.group(2)) 311 torrents.append(match.group(2))
312 else:
313 changing.append(match.group(2))
300 while(match.group(2) in new_files): 314 while(match.group(2) in new_files):
301 del new_files[new_files.index(match.group(2))] 315 del new_files[new_files.index(match.group(2))]
302 elif torrent_base != file_base and debug: 316 elif torrent_base != file_base and debug:
303 print "Not a torrent file:",f 317 print "Not a torrent file:",f
304 if match.group(1) == file_base: 318 if match.group(1) == file_base:
305 if match.group(3) and torrent_base != file_base and debug: 319 if match.group(3) and torrent_base != file_base and debug:
306 print "Torrent in file directory:",f 320 print "Torrent in file directory:",f
307 elif match.group(2) not in torrents: 321 elif match.group(2) not in torrents:
308 mystat = stat(f) 322 mystat = stat(f)
323 if mystat[ST_SIZE] < min_file_size:
324 pass # Too small ignore it.
309 if mystat[ST_SIZE]>0 and (time()-mystat[ST_MTIME])>minimum_age: 325 elif (time()-mystat[ST_MTIME]) > minimum_age:
310 new_files.append(match.group(2)) 326 new_files.append(match.group(2))
311 if debug: 327 if debug:
328 if new_files[-1] not in torrents:
329 if new_files[-1] not in changing:
330 if f not in noticed_newfiles:
312 print "New file:",f 331 print "New file:",f
332 noticed_newfiles.append(f)
333 else:
334 print "Changing file:",f
313 else: 335 else:
314 if debug: 336 if debug:
315 print "Actively modified: ",f 337 print "Actively modified: ",f
316 except Exception,e: 338 except Exception,e:
317 traceback.print_last()
318 print "--- IGNORING FILE:",t 339 print "--- IGNORING FILE: %s '%s'" % (f,e)
319 print 340 print
320 341
321 for t in threads.keys(): 342 for t in threads.keys():
322 # Remove threads that we don't have a bittorrent for. 343 # Remove threads that we don't have a bittorrent for.
323 if t not in torrents or not os.path.exists(file_base+t): 344 if t not in torrents or not os.path.exists(file_base+t):
… …
386 filename = "bt_daemon-%d.stats" % (os.getpid()) 407 filename = "bt_daemon-%d.stats" % (os.getpid())
387 makenew = 0 408 makenew = 0
388 writemsg("",filename,overwrite=makenew) 409 writemsg("",filename,overwrite=makenew)
389 writemsg("Stats: %s" % (ctime()) ,filename) 410 writemsg("Stats: %s" % (ctime()) ,filename)
390 writemsg("Seeding: %d" % (len(completed)) ,filename) 411 writemsg("Seeding: %d" % (len(completed)) ,filename)
391 writemsg("Fetching: %d (stalled: %d) (hashing: %d)" % (len(fetching),len(stalled),len(hashing)),filename) 412 writemsg("Fetching: %d/%d (stalled: %d) (hashing: %d)" % (len(fetching),max_fetches,len(stalled),len(hashing)),filename)
392 writemsg("Total up: %.3f MB" % (totalup) ,filename) 413 writemsg("Total up: %.3f MB" % (totalup) ,filename)
393 writemsg("UpRate: %.2f k/s" % (rateup) ,filename) 414 writemsg("UpRate: %.2f k/s" % (rateup) ,filename)
394 writemsg("UpMaxNow: %.2f k/s '%s'" % (max_up_rate,max_up_file) ,filename) 415 writemsg("UpMaxNow: %.2f k/s '%s'" % (max_up_rate,max_up_file) ,filename)
395 writemsg("UpAbsMax: %.2f k/s '%s'" % (absmax_upRate,absmax_upFile) ,filename) 416 writemsg("UpAbsMax: %.2f k/s '%s'" % (absmax_upRate,absmax_upFile) ,filename)
396 writemsg("DownRate: %.2f k/s" % (ratedown) ,filename) 417 writemsg("DownRate: %.2f k/s" % (ratedown) ,filename)
… …
408 stats_down.sort() 429 stats_down.sort()
409 writemsg(string.join(stats_down, "\n"), filename+".stats_down", overwrite=1, echo=0) 430 writemsg(string.join(stats_down, "\n"), filename+".stats_down", overwrite=1, echo=0)
410 431
411 for t in torrents: 432 for t in torrents:
412 # Start threads up if we have load free for them. Startup is harsh. 433 # Start threads up if we have load free for them. Startup is harsh.
434 if len(fetching) >= max_fetches:
435 continue
413 if t not in threads.keys() and (max_starting_threads - len(hashing) > 0): 436 if t not in threads.keys() and (max_starting_threads - len(hashing) > 0):
414 hashing.append(t) 437 hashing.append(t)
438 while t in noticed_newfiles:
439 del noticed_newfiles[noticed_newfiles.index(t)]
415 print "Starting torrent [%s]: %s" % (ctime(),t) 440 print "Starting torrent [%s]: %s" % (ctime(),t)
416 h = HeadlessDisplayer() 441 h = HeadlessDisplayer()
417 try: 442 try:
418 if not os.path.isdir(file_base+"/"+os.path.dirname(t)): 443 if not os.path.isdir(file_base+"/"+os.path.dirname(t)):
419 os.makedirs(file_base+"/"+os.path.dirname(t)) 444 os.makedirs(file_base+"/"+os.path.dirname(t))
… …
451 print "Failed to start thread: %s" % (t) 476 print "Failed to start thread: %s" % (t)
452 print "Reason: %s" % (str(e)) 477 print "Reason: %s" % (str(e))
453 478
454 if master_seed: 479 if master_seed:
455 for f in new_files: 480 for f in new_files:
481 if f in torrents:
482 print "in torrents:",f
483 continue
456 # Create the torrents for files that don't have them. 484 # Create the torrents for files that don't have them.
485 mystat = os.stat(file_base+"/"+f)
486 if mystat[ST_SIZE] < min_file_size:
487 print "Too small:",f
488 continue
489 # only add if it has a size and is not actively modified.
490 if (time() - mystat[ST_MTIME]) < minimum_age:
491 print "Too new:",f
492 continue
457 try: 493 try:
458 print "Creating torrent [%s]: %s" % (ctime(),f) 494 print "Creating torrent [%s]: %s" % (ctime(),f)
459 if not os.path.isdir(torrent_base+"/"+os.path.dirname(f)): 495 if not os.path.isdir(torrent_base+"/"+os.path.dirname(f)):
460 os.makedirs(torrent_base+"/"+os.path.dirname(f)) 496 os.makedirs(torrent_base+"/"+os.path.dirname(f))
461 make_meta_file(file_base+"/"+f,tracker,piece_size_pow2,progress=progress,comment=comment,target=torrent_base+"/"+f+".torrent") 497 make_meta_file(file_base+"/"+f,tracker,piece_size_pow2,progress=progress,comment=comment,target=torrent_base+"/"+f+".torrent")
462 except Exception, e: 498 except Exception, e:
463 print "Failed to create torrent: "+str(e) 499 print "Failed to create torrent: "+str(e)
464 500
465 sleep(cycle_sleep) 501 sleep(cycle_sleep)
466 except Exception, e: 502 except Exception, e:
467 import signal,traceback 503 sys.stderr.write("\n\nException caught. Terminating...\n%s\n\n" % e)
468 print e
469 traceback.print_stack()
470 os.kill(0,signal.SIGTERM) 504 os.kill(0,signal.SIGTERM)
Legend:
Removed from v.1.5  
changed lines
  Added in v.1.6
  ViewVC Help
Powered by ViewVC 1.1.13