#!/usr/bin/python -O
# $Id: bt_daemon.py,v 1.6 2004/06/16 08:46:48 carpaski Exp $
#
# Greatly modified by carpaski@gentoo.org
# btdownloadheadless written by Bram Cohen
# see LICENSE.txt for license information

import os,sys,string,signal
os.nice(5)

sys.path = ["/usr/lib/portage/pym"] + sys.path + ["/usr/bin"]
from portage import getconfig

import BitTorrent
from BitTorrent.download import download

from threading import Event,Thread
import types,re,random

from os import listdir, getcwd, stat
from stat import ST_SIZE,ST_MTIME
from os.path import abspath,isdir,isfile,normpath
from sys import argv, stdout, path
from cStringIO import StringIO
from time import time,ctime,sleep

from btmakemetafile import make_meta_file

def hours(n):
	if n == -1:
		return '<unknown>'
	if n == 0:
		return 'complete!'
	n = long(n)
	h, r = divmod(n, 60 * 60)
	m, sec = divmod(r, 60)
	if h > 1000000:
		return '<unknown>'
	if h > 0:
		return '%d hour %02d min %02d sec' % (h, m, sec)
	else:
		return '%d min %02d sec' % (m, sec)

class HeadlessDisplayer:
	def __init__(self):
		self.active = False
		self.done = False
		self.failed = False
		self.file = ''
		self.activity = ''
		self.percentDone = 0.0
		self.timeEst = 0
		self.downloadTo = ''
		self.downRate = 0.0
		self.upRate = 0.0
		self.downTotal = 0.0
		self.upTotal = 0
		self.errors = []
		self.last_update_time = 0

	def finished(self):
		self.done = True
		self.percentDone = 100
		self.timeEst = 0
		self.downRate = 0
		self.display({})

	def failed(self):
		self.done = True
		self.failed = True
		self.percentDone = 0
		self.timeEst = -1
		self.downRate = 0
		self.display({})

	def error(self, errormsg):
		self.errors.append(errormsg)
		self.display({})

	def display(self, dict):
		if ((self.last_update_time + 1) > time()):
			if dict.get('fractionDone') not in (0.0, 1.0):
				if not dict.has_key('activity'):
					return
		self.last_update_time = time()
		if dict.has_key('fractionDone'):
			self.percentDone = float(int(dict['fractionDone'] * 1000)) / 10
		if dict.has_key('timeEst'):
			self.timeEst = long(dict['timeEst'])
		if dict.has_key('activity') and not self.done:
			self.activity = dict['activity']
		if dict.has_key('downRate'):
			self.downRate = float(dict['downRate']) / (1 << 10)
			if self.downRate > 0:
				self.has_transfered = True
		if dict.has_key('upRate'):
			self.upRate = float(dict['upRate']) / (1 << 10)
			if self.upRate > 0:
				self.has_transfered = True
		if dict.has_key('upTotal'):
			self.upTotal = dict['upTotal']
		if dict.has_key('downTotal'):
			self.active = True
			self.downTotal = dict['downTotal']
		for x in self.errors:
			print self.file+": "+str(x)
		self.errors = []
		return

	def chooseFile(self, default, size, saveas, dir):
		self.file = '%s (%.1f MB)' % (default, float(size) / (1 << 20))
		if saveas != '':
			default = saveas
		self.downloadTo = abspath(default)
		return default

	def newpath(self, path):
		self.downloadTo = path

def prefix_array(array,prefix,doblanks=1):
	"""Prepends a given prefix to each element in an Array/List/Tuple.
	Returns a List."""
	if type(array) not in [types.ListType, types.TupleType]:
		raise TypeError, "List or Tuple expected. Got %s" % type(array)
	newarray=[]
	for x in array:
		if x or doblanks:
			newarray.append(prefix + x)
		else:
			newarray.append(x)
	return newarray

def list_files(dirlist):
	file_list = []
	for mydir in dirlist:
		mys = mydir.split("/")
		if mys[-1][0] == '.':
			continue
		if isdir(mydir):
			try:
				file_list += list_files(prefix_array(listdir(mydir),mydir+"/"))
			except Exception,e:
				print e
		elif isfile(mydir):
			file_list.append(mydir)
		else:
			print "Unknown type:",mydir
	return file_list


def progress(amount):
	# Dummy function for the create-torrent progress meter
	pass


def writemsg(msg,fileName,overwrite=0,create=0,echo=1):
	if overwrite or create:
		fileHandle = open(fileName, "w")
	else:
		fileHandle = open(fileName, "a")

	fileHandle.write(msg+"\n")
	fileHandle.flush()
	fileHandle.close()
	if echo:
		print msg


# Useless variables
cols = 80

# This is 2^x ... put the x below -- Chunk size for sha1 -- 18 == 256k
piece_size_pow2        =      18
max_starting_threads   =       1
loop_mod               =    4320 # Rotating value (for logs)
rotate_logs            =       0 # Make one log or roate into loop_mod logs
cycle_sleep            =      20
spawn_thread_delay     =       0
debug                  =       0
minimum_age            =     120
port_min               =    6881
port_max               =    7280
check_hashes           =       0
shuffle_files          =       0
master_seed            =       0
min_uploads            =      20
max_uploads            =      20
download_slice_size    =   65536
max_slice_length       =  131072
request_backlog        =      20
timeout                =      90
timeout_check_interval =      20
stats                  =       1
min_file_size          = 5000000
max_fetches            =       3

# No trailing /
file_base    = "/usr/portage/distfiles"
torrent_base = "/tmp/torrents"

comment = "Gentoo Linux BitTorrent Mirror System"
tracker = "http://egret.gentoo.org:6969/announce"

if __name__ == '__main__':
	try:
		mydict = getconfig("bt_daemon.conf")
		if not mydict:
			print "bt_daemon.conf is missing"
	
		if mydict.has_key("file_base") and mydict["file_base"]:
			file_base = os.path.normpath(mydict["file_base"])
		else:
			print "file_base not specified in config."
			sys.exit(1)
	
		if mydict.has_key("torrent_base") and mydict["torrent_base"]:
			torrent_base = os.path.normpath(mydict["torrent_base"])
		else:
			print "torrent_base not specified in config."
			sys.exit(1)
	
		threads   = {}
		dirs = [torrent_base[:],file_base[:]]
		re_file = re.compile("("+torrent_base[:]+"|"+file_base[:]+")(/.+?)(\.torrent)?$")
		absmax_upRate = 0.0
		absmax_upFile = ''

		noticed_newfiles = []
	
		loop_val = 0
		while(True):
			mydict = getconfig("bt_daemon.conf")
			if not mydict:
				sys.stderr.write("bt_daemon.conf is missing or contains errors\n")
	
			if mydict.has_key("master_seed"):
				master_seed = int(mydict["master_seed"])
	
			if mydict.has_key("piece_size_pow2"):
				piece_size_pow2 = int(mydict["piece_size_pow2"])
			if mydict.has_key("max_starting_threads"):
				max_starting_threads = int(mydict["max_starting_threads"])
			if mydict.has_key("max_fetches"):
				max_fetches = int(mydict["max_fetches"])
			if mydict.has_key("loop_mod"):
				loop_mod = int(mydict["loop_mod"])
			if mydict.has_key("rotate_logs"):
				rotate_logs = int(mydict["rotate_logs"])
			if mydict.has_key("cycle_sleep"):
				cycle_sleep = int(mydict["cycle_sleep"])
			if mydict.has_key("spawn_thread_delay"):
				spawn_thread_delay = int(mydict["spawn_thread_delay"])
			if mydict.has_key("shuffle_files"):
				shuffle_files = int(mydict["shuffle_files"])
			if mydict.has_key("debug"):
				debug = int(mydict["debug"])

			if mydict.has_key("min_uploads"):
				min_uploads = int(mydict["min_uploads"])
			if mydict.has_key("max_uploads"):
				max_uploads = int(mydict["max_uploads"])
			if mydict.has_key("download_slice_size"):
				download_slice_size = int(mydict["download_slice_size"])
			if mydict.has_key("max_slice_length"):
				max_slice_length = int(mydict["max_slice_length"])
			if mydict.has_key("request_backlog"):
				request_backlog = int(mydict["request_backlog"])
			if mydict.has_key("timeout"):
				timeout = int(mydict["timeout"])
			if mydict.has_key("timeout_check_interval"):
				timeout_check_interval = int(mydict["timeout_check_interval"])

			if mydict.has_key("comment"):
				comment = mydict["comment"]
			if mydict.has_key("port_min"):
				port_min = mydict["port_min"]
			if mydict.has_key("port_max"):
				port_max = mydict["port_max"]
			if mydict.has_key("check_hashes"):
				check_hashes = mydict["check_hashes"]
	
			if mydict.has_key("tracker") and mydict["tracker"]:
				tracker = mydict["tracker"]
			else:
				print "tracker not specified in config."
				sys.exit(1)
	
	
			loop_val = (loop_val + 1) % loop_mod
			files = list_files(dirs)
			files.sort()
			new_files = []
			torrents  = []
			changing  = []
	
			if shuffle_files:
				random.shuffle(files)
	
			for f in files:
				try:
					match = re_file.match(f)
					if not match:
						print "Failed to match:",f
					else:
						if match.group(1) == torrent_base:
							if match.group(3): # It's a torrent, yay!
								if match.group(2) not in torrents:
									mystat = stat(f)
									# only add if it has a size and is not actively modified.
									if mystat[ST_SIZE]>0 and (time()-mystat[ST_MTIME])>minimum_age:
										torrents.append(match.group(2))
									else:
										changing.append(match.group(2))
									while(match.group(2) in new_files):
										del new_files[new_files.index(match.group(2))]
							elif torrent_base != file_base and debug:
								print "Not a torrent file:",f
						if match.group(1) == file_base:
							if match.group(3) and torrent_base != file_base and debug:
								print "Torrent in file directory:",f
							elif match.group(2) not in torrents:
								mystat = stat(f)
								if mystat[ST_SIZE] < min_file_size:
									pass # Too small ignore it.
								elif (time()-mystat[ST_MTIME]) > minimum_age:
										new_files.append(match.group(2))
										if debug:
											if new_files[-1] not in torrents:
												if new_files[-1] not in changing:
													if f not in noticed_newfiles:
														print "New file:",f
														noticed_newfiles.append(f)
												else:
													print "Changing file:",f
								else:
									if debug:
										print "Actively modified: ",f
				except Exception,e:
					print "--- IGNORING FILE: %s '%s'" % (f,e)
					print
	
			for t in threads.keys():
				# Remove threads that we don't have a bittorrent for.
				if t not in torrents or not os.path.exists(file_base+t):
					print "Stopping torrent [%s]: %s" % (ctime(),t)
					threads[t][0]._Thread__stop()
					del threads[t]
					while(t in torrents):
						del torrents[torrents.index(t)]
					
				elif not threads[t][0].isAlive():
					print "Thread died [%s]: %s" % (ctime(),t)
					del threads[t]
			
			completed    = []
			fetching     = []
			hashing      = []
			stalled      = []
	
			stats_up     = []
			stats_down   = []
	
			totalup      = 0
			rateup       = 0
			ratedown     = 0
	
			max_up_rate = 0
			max_up_file = ''
			max_down_rate = 0
			max_down_file = ''
			completed_string = ""
			for t in threads.keys():
				# Let's do some stats.
				if not threads[t][1].failed and threads[t][0].isAlive():
					if threads[t][1].done:
						completed.append(t)
					else:
						fetching.append(t)
						if not threads[t][1].active:
							hashing.append(t)
						elif threads[t][1].downRate == 0:
							stalled.append(t)
	
					stats_up.append("%s\tT:%f\tR:%03.2f\t%s" % (t,threads[t][1].upTotal,threads[t][1].upRate,threads[t][1].activity))
					stats_down.append("%s\tT:%f\tR:%03.2f\t%s" % (t,threads[t][1].percentDone,threads[t][1].downRate,threads[t][1].activity))
	
					totalup  += threads[t][1].upTotal
	
					rateup   += threads[t][1].upRate
					if threads[t][1].upRate > max_up_rate:
						max_up_rate = threads[t][1].upRate
						max_up_file = threads[t][1].file[:]
					if threads[t][1].upRate > absmax_upRate:
						absmax_upRate = threads[t][1].upRate
						absmax_upFile = threads[t][1].file[:]
					
					ratedown += threads[t][1].downRate
					if threads[t][1].downRate > max_down_rate:
						max_down_rate = threads[t][1].downRate
						max_down_file = threads[t][1].file[:]
			
			if stats:
				if rotate_logs:
					filename = "bt_daemon-%d-%04d.stats" % (os.getpid(),loop_val)
					makenew  = 1
				else:
					filename = "bt_daemon-%d.stats" % (os.getpid())
					makenew  = 0
				writemsg("",filename,overwrite=makenew)
				writemsg("Stats:    %s"                             % (ctime())                     ,filename)
				writemsg("Seeding:  %d"                             % (len(completed))              ,filename)
				writemsg("Fetching: %d/%d (stalled: %d) (hashing: %d)" % (len(fetching),max_fetches,len(stalled),len(hashing)),filename)
				writemsg("Total up: %.3f MB"                        % (totalup)                     ,filename)
				writemsg("UpRate:   %.2f k/s"                       % (rateup)                      ,filename)
				writemsg("UpMaxNow: %.2f k/s '%s'"                  % (max_up_rate,max_up_file)     ,filename)
				writemsg("UpAbsMax: %.2f k/s '%s'"                  % (absmax_upRate,absmax_upFile) ,filename)
				writemsg("DownRate: %.2f k/s"                       % (ratedown)                    ,filename)
				writemsg("DownBest: %.2f k/s '%s'"                  % (max_down_rate,max_down_file) ,filename)
				writemsg("",filename)
				
				completed.sort()
				writemsg(string.join(completed,  "\n"), filename+".completed",  overwrite=1, echo=0)
				hashing.sort()
				writemsg(string.join(hashing,    "\n"), filename+".hashing",    overwrite=1, echo=0)
				stalled.sort()
				writemsg(string.join(stalled,    "\n"), filename+".stalled",    overwrite=1, echo=0)
				stats_up.sort()
				writemsg(string.join(stats_up,   "\n"), filename+".stats_up",   overwrite=1, echo=0)
				stats_down.sort()
				writemsg(string.join(stats_down, "\n"), filename+".stats_down", overwrite=1, echo=0)
	
			for t in torrents:
				# Start threads up if we have load free for them. Startup is harsh.
				if len(fetching) >= max_fetches:
					continue
				if t not in threads.keys() and (max_starting_threads - len(hashing) > 0):
					hashing.append(t)
					while t in noticed_newfiles:
						del noticed_newfiles[noticed_newfiles.index(t)]
					print "Starting torrent [%s]: %s" % (ctime(),t)
					h = HeadlessDisplayer()
					try:
						if not os.path.isdir(file_base+"/"+os.path.dirname(t)):
							os.makedirs(file_base+"/"+os.path.dirname(t))
						download_args = [
							# responsefile -> Torrent filename
							'--responsefile', torrent_base+"/"+t+".torrent",
							# saveas -> Real filename
							'--saveas', file_base+"/"+t,
							# max_slice_length -> How much we can send in a single request
							'--max_slice_length', str(max_slice_length),
							# timeout_check_interval -> Time between timeout checks
							'--timeout_check_interval', str(timeout_check_interval),
							# timeout -> How long until we time out a connection
							'--timeout', str(timeout),
							# minport -> Lowest port allowed for files
							'--minport', str(port_min),
							# maxport -> Highest port allowed for files
							'--maxport', str(port_max),
							# request_backlog -> How many requests per pipe do we hold
							'--request_backlog', str(request_backlog),
							# download_slice_size -> How much do we request?
							'--download_slice_size', str(download_slice_size),
							# max_uploads -> How many do we serve out?
							'--max_uploads', str(max_uploads),
							# min_uploads -> When do we switch from random to rare?
							'--min_uploads', str(min_uploads),
							# check_hashes -> Do we check the on-disk files before joining?
							'--check_hashes', str(check_hashes)
							]
						t_thread = Thread(target=download,name="BitTorrent::"+t,args=(download_args, h.chooseFile, h.display, h.finished, h.error, Event(), cols))
						threads[t] = [t_thread,h]
						t_thread.start()
						sleep(spawn_thread_delay)
					except Exception, e:
						print "Failed to start thread: %s" % (t)
						print "Reason: %s" % (str(e))
	
			if master_seed:
				for f in new_files:
					if f in torrents:
						print "in torrents:",f
						continue
					# Create the torrents for files that don't have them.
					mystat = os.stat(file_base+"/"+f)
					if mystat[ST_SIZE] < min_file_size:
						print "Too small:",f
						continue
					# only add if it has a size and is not actively modified.
					if (time() - mystat[ST_MTIME]) < minimum_age:
						print "Too new:",f
						continue
					try:
						print "Creating torrent [%s]: %s" % (ctime(),f)
						if not os.path.isdir(torrent_base+"/"+os.path.dirname(f)):
							os.makedirs(torrent_base+"/"+os.path.dirname(f))
						make_meta_file(file_base+"/"+f,tracker,piece_size_pow2,progress=progress,comment=comment,target=torrent_base+"/"+f+".torrent")
					except Exception, e:
						print "Failed to create torrent: "+str(e)
			
			sleep(cycle_sleep)
	except Exception, e:
		sys.stderr.write("\n\nException caught. Terminating...\n%s\n\n" % e)
		os.kill(0,signal.SIGTERM)