src/packages/ebuilddb.py

#!/usr/bin/python -O

__revision__ = '$Revision: 1.8 $'
# $Source: /var/cvsroot/gentoo/src/packages/ebuilddb.py,v $

import config
import sys
import os
import time
import changelogs
import MySQLdb

def db_connect():
    return MySQLdb.connect(host = config.HOST,
        user = config.USER,
        passwd = config.PASSWD,
        db = config.DATABASE)

def find_ebuilds():
    #print "walking..."
    # yeah, i know we can os.path.walk, but this runs faster ;-)
    pipe = os.popen("find %s -name '*.ebuild'" % config.PORTAGE_DIR)
    s = pipe.read()
    pipe.close()
    return s.split()

def parse_ebuild(s, pkgsplit):
    """Parse ebuild info based on path name"""
    parsed = {}
    s=s.split('/')
    parsed['category'] = s[-3]
    package = s[-1].split('.ebuild')[0]
    pieces = pkgsplit(package)
    if not pieces:
        return None
    parsed['name'] = pieces[0]
    if pieces[2] == 'r0':
        parsed['version'] = pieces[1]
    else:
        parsed['version'] = '-'.join(pieces[1:])

    return parsed

def get_ebuild_record(db,ebinfo):
    c = db.cursor()
    query = ('SELECT * FROM ebuild WHERE category="%s" AND name="%s" '
        'AND version="%s" LIMIT 1' % (ebinfo['category'], ebinfo['name'],
        ebinfo['version']))
    c.execute(query)
    result = c.fetchone()
    return result

def create_ebuild_record(db,ebinfo):
    c = db.cursor()
    d = db.cursor()
    # update the package table

    # this is a really ugly dict comprehension
    escaped = {}
    for item in ebinfo.items():
        x, y = item
        if type(y) is str:
            y = MySQLdb.escape_string(y)
        escaped[x] = y

    query=('INSERT INTO package SET category="%(category)s",name="%(name)s",'
        'homepage="%(homepage)s",description="%(description)s",'
        'license="%(license)s"' % escaped)
    try:
        c.execute(query)
    except MySQLdb.cursors.IntegrityError:
        # duplicate key (we hope)
        query = ('REPLACE INTO package VALUES ("%(category)s","%(name)s",'
            '"%(homepage)s","%(description)s","%(license)s",0)' % escaped)
        c.execute(query)

    # then add particular ebuild
    query = ('INSERT INTO ebuild VALUES ("%(category)s","%(name)s",'
        '"%(version)s",%(time)s,"%(archs)s","%(changelog)s","",%(masked)d)'
        % escaped)
    d.execute(query)

def update_ebuild_record(db,ebinfo):
    c = db.cursor()
    escaped = {}
    for item in ebinfo.items():
        x, y = item
        if type(y) is str:
            y = MySQLdb.escape_string(y)
        escaped[x] = y

    query="""REPLACE INTO package VALUES ("%(category)s","%(name)s",\
        "%(homepage)s","%(description)s","%(license)s",0);""" % escaped
    c.execute(query)

    query = ('UPDATE ebuild '
        'SET when_found="%(time)s",'
        'arch="%(archs)s",'
        'changelog="%(changelog)s",'
        'prevarch="%(prevarch)s", '
        'is_masked=%(masked)d '
        'WHERE category="%(category)s" '
        'AND name="%(name)s" '
        'AND version="%(version)s" ' % escaped)
    try:
        c.execute(query)
    except MySQLdb.MySQLError, data:
        print 'error occurred: '
        print 'query: %s' % query
        print 'error: %s' % data


def get_extended_info(ebuild):
    filename = os.path.join(config.PORTAGE_DIR,'metadata/cache',
        ebuild['category'],'%s-%s' % (ebuild['name'],ebuild['version']))
    try:
        lines = open(filename,'r').readlines()
    except IOError:
        print 'Error opening %s' % filename
        lines = []
    lines = [ s.strip() for s in lines ]
    try:
        ebuild['archs'] = ','.join(lines[8].split())
    except IndexError:
        ebuild['archs'] = ''
    try:
        ebuild['homepage'] = lines[5]
    except IndexError:
        ebuild['homepage'] = 'http://www.gentoo.org/'
    try:
        ebuild['license'] = lines[6]
    except IndexError:
        ebuild['license'] = ''
    try:
        ebuild['description'] = lines[7]
    except IndexError:
        ebuild['description'] = ''
    return ebuild

def get_mtime(s):
    """Get mtime of file, return in format that MySQL would like"""
    try:
        mtime = os.path.getmtime(s)
    except OSError:
        return 'NULL'
    strftime = time.strftime("%Y%m%d%H%M%S",time.localtime(mtime))
    return strftime

def is_masked(tree, ebuild):
    """Return true if packages is masked in tree"""
    return (not tree.visible(['%(category)s/%(name)s-%(version)s' % ebuild]))

def main(argv = []):
    """Main program entry point"""
    # We need to "fake" as repoman so portage will ignore local system
    # settings
    os.environ['PORTAGE_CALLER'] = 'repoman'

    ebuilds = find_ebuilds()
    db = db_connect()

    sys.path = ["/usr/lib/portage/pym"]+sys.path
    import portage
    # ... and then wait like a f**king hour
    sys.path = sys.path[1:]

    # We want to use the base profile, not any system-specific one
    p_config = portage.config(config_profile_path='/usr/portage/profiles/base',
            local_config=False)

    tree = portage.portdbapi('/usr/portage', p_config)

    # are we updating the db or just rebuilding it?
    rebuild = len(argv) == 2 and argv[1] == 'rebuild'
    for s in ebuilds:
        fields = parse_ebuild(s, portage.pkgsplit)
        if not fields:
            continue
        result = get_ebuild_record(db,fields)
        fields = get_extended_info(fields)
        fields['changelog'] = changelogs.changelog('%s/ChangeLog'
            % os.path.dirname(s))
        fields['time'] = get_mtime(s)
        fields['masked'] = int(is_masked(tree, fields))

        if not (result or rebuild):
            create_ebuild_record(db,fields)
        elif result and rebuild:
            fields['prevarch'] = result[6]
            if ' ' in fields['prevarch']:
                # old db layout
                fields['prevarch'] = ','.join(fields['prevarch'].split())
            fields['arch'] = [4]
            update_ebuild_record(db, fields)
        elif rebuild:
            pass
        elif result[4].split(',') != fields['archs'].split(','):
            #print 'ebuild archs=',fields['archs']
            #print 'db archs=',result[4]
            #print
            # keywords change, update db
            fields['prevarch'] = result[4]
            update_ebuild_record(db,fields)
        elif result[7] != fields['masked']:
            #print 'mask changed for %s-%s' % (fields['name'], fields['version'])
            #print fields['masked']
            fields['prevarch'] = result[4]
            update_ebuild_record(db,fields)

    db.commit()
if __name__ == '__main__':
    sys.exit(main(sys.argv))
1	marduk	1.4	#!/usr/bin/python -O
2
3	marduk	1.8.2.1	__revision__ = '$Revision: 1.8 $'
4	marduk	1.5	# $Source: /var/cvsroot/gentoo/src/packages/ebuilddb.py,v $
5	marduk	1.1
6			import config
7			import sys
8			import os
9			import time
10			import changelogs
11			import MySQLdb
12
13			def db_connect():
14			return MySQLdb.connect(host = config.HOST,
15			user = config.USER,
16			passwd = config.PASSWD,
17			db = config.DATABASE)
18	marduk	1.5
19	marduk	1.1	def find_ebuilds():
20			#print "walking..."
21			# yeah, i know we can os.path.walk, but this runs faster ;-)
22			pipe = os.popen("find %s -name '*.ebuild'" % config.PORTAGE_DIR)
23			s = pipe.read()
24			pipe.close()
25			return s.split()
26	marduk	1.5
27	marduk	1.4	def parse_ebuild(s, pkgsplit):
28	marduk	1.1	"""Parse ebuild info based on path name"""
29			parsed = {}
30			s=s.split('/')
31			parsed['category'] = s[-3]
32			package = s[-1].split('.ebuild')[0]
33	marduk	1.4	pieces = pkgsplit(package)
34	marduk	1.3	if not pieces:
35			return None
36			parsed['name'] = pieces[0]
37			if pieces[2] == 'r0':
38			parsed['version'] = pieces[1]
39			else:
40			parsed['version'] = '-'.join(pieces[1:])
41	marduk	1.1
42			return parsed
43
44			def get_ebuild_record(db,ebinfo):
45			c = db.cursor()
46	marduk	1.5	query = ('SELECT * FROM ebuild WHERE category="%s" AND name="%s" '
47	marduk	1.1	'AND version="%s" LIMIT 1' % (ebinfo['category'], ebinfo['name'],
48			ebinfo['version']))
49			c.execute(query)
50			result = c.fetchone()
51			return result
52
53			def create_ebuild_record(db,ebinfo):
54			c = db.cursor()
55			d = db.cursor()
56			# update the package table
57
58			# this is a really ugly dict comprehension
59	marduk	1.4	escaped = {}
60			for item in ebinfo.items():
61			x, y = item
62			if type(y) is str:
63			y = MySQLdb.escape_string(y)
64			escaped[x] = y
65	marduk	1.5
66	marduk	1.4	query=('INSERT INTO package SET category="%(category)s",name="%(name)s",'
67			'homepage="%(homepage)s",description="%(description)s",'
68			'license="%(license)s"' % escaped)
69			try:
70			c.execute(query)
71			except MySQLdb.cursors.IntegrityError:
72			# duplicate key (we hope)
73			query = ('REPLACE INTO package VALUES ("%(category)s","%(name)s",'
74			'"%(homepage)s","%(description)s","%(license)s",0)' % escaped)
75			c.execute(query)
76	marduk	1.1
77			# then add particular ebuild
78			query = ('INSERT INTO ebuild VALUES ("%(category)s","%(name)s",'
79	marduk	1.5	'"%(version)s",%(time)s,"%(archs)s","%(changelog)s","",%(masked)d)'
80	marduk	1.1	% escaped)
81			d.execute(query)
82	marduk	1.5
83	marduk	1.1	def update_ebuild_record(db,ebinfo):
84			c = db.cursor()
85	marduk	1.4	escaped = {}
86			for item in ebinfo.items():
87			x, y = item
88			if type(y) is str:
89			y = MySQLdb.escape_string(y)
90			escaped[x] = y
91	marduk	1.5
92	marduk	1.3	query="""REPLACE INTO package VALUES ("%(category)s","%(name)s",\
93	marduk	1.4	"%(homepage)s","%(description)s","%(license)s",0);""" % escaped
94	marduk	1.3	c.execute(query)
95	marduk	1.5
96	marduk	1.1	query = ('UPDATE ebuild '
97			'SET when_found="%(time)s",'
98			'arch="%(archs)s",'
99			'changelog="%(changelog)s",'
100	marduk	1.4	'prevarch="%(prevarch)s", '
101			'is_masked=%(masked)d '
102	marduk	1.1	'WHERE category="%(category)s" '
103			'AND name="%(name)s" '
104			'AND version="%(version)s" ' % escaped)
105	marduk	1.3	try:
106			c.execute(query)
107			except MySQLdb.MySQLError, data:
108			print 'error occurred: '
109			print 'query: %s' % query
110			print 'error: %s' % data
111	marduk	1.5
112	marduk	1.1
113			def get_extended_info(ebuild):
114			filename = os.path.join(config.PORTAGE_DIR,'metadata/cache',
115			ebuild['category'],'%s-%s' % (ebuild['name'],ebuild['version']))
116			try:
117			lines = open(filename,'r').readlines()
118			except IOError:
119	marduk	1.5	print 'Error opening %s' % filename
120	marduk	1.1	lines = []
121			lines = [ s.strip() for s in lines ]
122			try:
123	marduk	1.8	ebuild['archs'] = ','.join(lines[8].split())
124	marduk	1.1	except IndexError:
125			ebuild['archs'] = ''
126			try:
127			ebuild['homepage'] = lines[5]
128			except IndexError:
129			ebuild['homepage'] = 'http://www.gentoo.org/'
130			try:
131			ebuild['license'] = lines[6]
132			except IndexError:
133			ebuild['license'] = ''
134			try:
135			ebuild['description'] = lines[7]
136			except IndexError:
137			ebuild['description'] = ''
138			return ebuild
139
140			def get_mtime(s):
141			"""Get mtime of file, return in format that MySQL would like"""
142			try:
143	marduk	1.7	mtime = os.path.getmtime(s)
144	marduk	1.5	except OSError:
145	marduk	1.1	return 'NULL'
146	marduk	1.7	strftime = time.strftime("%Y%m%d%H%M%S",time.localtime(mtime))
147			return strftime
148	marduk	1.1
149	marduk	1.4	def is_masked(tree, ebuild):
150			"""Return true if packages is masked in tree"""
151			return (not tree.visible(['%(category)s/%(name)s-%(version)s' % ebuild]))
152
153	marduk	1.7	def main(argv = []):
154	marduk	1.6	"""Main program entry point"""
155			# We need to "fake" as repoman so portage will ignore local system
156			# settings
157			os.environ['PORTAGE_CALLER'] = 'repoman'
158
159	marduk	1.1	ebuilds = find_ebuilds()
160			db = db_connect()
161	marduk	1.5
162	marduk	1.4	sys.path = ["/usr/lib/portage/pym"]+sys.path
163			import portage
164			# ... and then wait like a f**king hour
165			sys.path = sys.path[1:]
166
167			# We want to use the base profile, not any system-specific one
168	marduk	1.8.2.1	p_config = portage.config(config_profile_path='/usr/portage/profiles/base',
169			local_config=False)
170	marduk	1.5
171			tree = portage.portdbapi('/usr/portage', p_config)
172
173	marduk	1.7	# are we updating the db or just rebuilding it?
174			rebuild = len(argv) == 2 and argv[1] == 'rebuild'
175	marduk	1.1	for s in ebuilds:
176	marduk	1.4	fields = parse_ebuild(s, portage.pkgsplit)
177	marduk	1.3	if not fields:
178	marduk	1.1	continue
179			result = get_ebuild_record(db,fields)
180			fields = get_extended_info(fields)
181	marduk	1.5	fields['changelog'] = changelogs.changelog('%s/ChangeLog'
182	marduk	1.1	% os.path.dirname(s))
183			fields['time'] = get_mtime(s)
184	marduk	1.4	fields['masked'] = int(is_masked(tree, fields))
185	marduk	1.5
186	marduk	1.7	if not (result or rebuild):
187	marduk	1.1	create_ebuild_record(db,fields)
188	marduk	1.7	elif result and rebuild:
189			fields['prevarch'] = result[6]
190	marduk	1.8	if ' ' in fields['prevarch']:
191			# old db layout
192			fields['prevarch'] = ','.join(fields['prevarch'].split())
193	marduk	1.7	fields['arch'] = [4]
194			update_ebuild_record(db, fields)
195	marduk	1.8	elif rebuild:
196			pass
197			elif result[4].split(',') != fields['archs'].split(','):
198	marduk	1.1	#print 'ebuild archs=',fields['archs']
199			#print 'db archs=',result[4]
200			#print
201			# keywords change, update db
202			fields['prevarch'] = result[4]
203			update_ebuild_record(db,fields)
204	marduk	1.4	elif result[7] != fields['masked']:
205			#print 'mask changed for %s-%s' % (fields['name'], fields['version'])
206			#print fields['masked']
207			fields['prevarch'] = result[4]
208			update_ebuild_record(db,fields)
209	marduk	1.5
210			db.commit()
211	marduk	1.1	if __name__ == '__main__':
212	marduk	1.7	sys.exit(main(sys.argv))